thumbgate 1.29.2 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +54 -16
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +143 -14
- package/adapters/opencode/opencode.json +1 -1
- package/bench/observability-eval-suite.json +2 -2
- package/bin/cli.js +154 -36
- package/config/evals/generation-quality-golden.json +95 -0
- package/config/evals/rag-answer-quality-golden.json +91 -0
- package/config/evals/retrieval-hybrid-ablation.json +66 -0
- package/config/evals/retrieval-ranking-golden.json +522 -0
- package/config/gates/default.json +217 -50
- package/config/mcp-allowlists.json +215 -185
- package/config/model-tiers.json +7 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/glama.json +6 -0
- package/package.json +94 -11
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +22 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/hero-thumbs.svg +68 -0
- package/public/assets/diagrams/loop.svg +40 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/self-improving-thumbs-loop.svg +105 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/compare.html +1 -0
- package/public/dashboard.html +126 -28
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +143 -13
- package/public/numbers.html +3 -2
- package/public/pricing.html +143 -30
- package/public/whitepaper.html +189 -0
- package/scripts/a-plus-evidence-scorecard.js +303 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/async-eval-observability.js +36 -11
- package/scripts/audit-trail.js +37 -1
- package/scripts/auto-promote-gates.js +149 -34
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/colbert-style-maxsim.js +236 -0
- package/scripts/cross-encoder-reranker.js +359 -126
- package/scripts/dashboard-chat.js +350 -17
- package/scripts/document-intake.js +283 -7
- package/scripts/eval-quality-suite.js +204 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +359 -189
- package/scripts/feedback-paths.js +32 -13
- package/scripts/feedback-quality.js +53 -0
- package/scripts/filesystem-search.js +17 -7
- package/scripts/gates-engine.js +98 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/harness-tool-names.js +70 -0
- package/scripts/hook-runtime.js +10 -3
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-db.js +16 -5
- package/scripts/lesson-embedding-index.js +67 -20
- package/scripts/lesson-embedding-maintenance.js +177 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-reranker.js +55 -9
- package/scripts/lesson-retrieval.js +375 -32
- package/scripts/lesson-search.js +48 -11
- package/scripts/llm-client.js +304 -15
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/model-tier-router.js +593 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/pragmatic-hybrid-search.js +379 -0
- package/scripts/published-cli.js +11 -1
- package/scripts/rag-document-pipeline.js +461 -0
- package/scripts/rag-structured-output.js +441 -0
- package/scripts/ragas-style-metrics.js +351 -0
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/request-envelope.js +178 -0
- package/scripts/rerank-pipeline.js +370 -0
- package/scripts/rerank-quality-eval.js +155 -0
- package/scripts/retrieval-hybrid-ablation.js +120 -0
- package/scripts/retrieval-quality-tier.js +118 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/secret-scanner.js +395 -4
- package/scripts/self-distill-agent.js +7 -1
- package/scripts/self-healing-check.js +25 -0
- package/scripts/skill-packs.js +183 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/statusline-links.js +1 -1
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/statusline.sh +8 -1
- package/scripts/telemetry-analytics.js +13 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/thumbgate-search.js +98 -6
- package/scripts/tier-budget-guard.js +186 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +95 -1
- package/scripts/vector-store.js +108 -4
- package/scripts/verify-marketing-pages-deployed.js +85 -3
- package/server.json +44 -0
- package/smithery.yaml +17 -0
- package/src/api/server.js +424 -99
|
@@ -78,6 +78,15 @@ function isTransientProjectDir(dirPath, options = {}) {
|
|
|
78
78
|
if (!normalizedDir) return true;
|
|
79
79
|
if (!dirExists(normalizedDir)) return true;
|
|
80
80
|
|
|
81
|
+
// MCP hosts and desktop launchers commonly start global servers from `/` or
|
|
82
|
+
// the user's home directory. Those are launcher contexts, not projects. If
|
|
83
|
+
// they win resolution they create `projects/default` / `~/.thumbgate` split
|
|
84
|
+
// stores and make the same lesson corpus depend on which client started the
|
|
85
|
+
// process. A durable active-project state is a better signal.
|
|
86
|
+
if (normalizedDir === path.parse(normalizedDir).root) return true;
|
|
87
|
+
const homeDir = normalizeDir(getHomeDir(options));
|
|
88
|
+
if (homeDir && normalizedDir === homeDir) return true;
|
|
89
|
+
|
|
81
90
|
const runtimeDir = getRuntimeDir(options);
|
|
82
91
|
if (isWithinDir(normalizedDir, runtimeDir)) return true;
|
|
83
92
|
|
|
@@ -119,11 +128,12 @@ function writeActiveProjectState(projectDir, options = {}) {
|
|
|
119
128
|
function resolveProjectDir(options = {}) {
|
|
120
129
|
const env = options.env || process.env;
|
|
121
130
|
const stored = options.includeStored === false ? null : readActiveProjectState(options);
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
131
|
+
// An injected cwd is authoritative for callers that resolve on behalf of a
|
|
132
|
+
// different process. Mixing in this Node process's cwd can silently route a
|
|
133
|
+
// global launcher back into ThumbGate's own checkout during diagnostics.
|
|
134
|
+
const cwdCandidates = uniquePaths(options.cwd
|
|
135
|
+
? [options.cwd]
|
|
136
|
+
: [env.PWD, process.cwd()]);
|
|
127
137
|
const isTransientExecution = cwdCandidates.length > 0
|
|
128
138
|
&& cwdCandidates.every((candidate) => isTransientProjectDir(candidate, options));
|
|
129
139
|
const candidates = uniquePaths([
|
|
@@ -167,15 +177,12 @@ function getExplicitFeedbackDir(options = {}) {
|
|
|
167
177
|
const env = options.env || process.env;
|
|
168
178
|
if (options.feedbackDir) return options.feedbackDir;
|
|
169
179
|
if (options.skipExplicitFeedbackDir) return null;
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
if (env.THUMBGATE_FEEDBACK_DIR
|
|
180
|
+
// An explicit storage root is the strongest storage instruction. Project
|
|
181
|
+
// metadata may still identify the project, but it must not redirect writes
|
|
182
|
+
// out of an isolated test/runtime directory.
|
|
183
|
+
if (env.THUMBGATE_FEEDBACK_DIR) {
|
|
174
184
|
return env.THUMBGATE_FEEDBACK_DIR;
|
|
175
185
|
}
|
|
176
|
-
if (hasDirectProjectScope(options)) {
|
|
177
|
-
return null;
|
|
178
|
-
}
|
|
179
186
|
if (env.RAILWAY_VOLUME_MOUNT_PATH) {
|
|
180
187
|
return path.join(env.RAILWAY_VOLUME_MOUNT_PATH, 'feedback');
|
|
181
188
|
}
|
|
@@ -221,7 +228,19 @@ function resolveFeedbackDir(options = {}) {
|
|
|
221
228
|
const localLegacy = getLegacyFeedbackDir(options);
|
|
222
229
|
if (dirExists(localLegacy)) return localLegacy;
|
|
223
230
|
|
|
224
|
-
|
|
231
|
+
// Existing installations may only have the pre-1.31 basename-scoped store.
|
|
232
|
+
// Preserve that corpus instead of silently switching to an empty directory.
|
|
233
|
+
// New projects never create this legacy global shape, so same-basename
|
|
234
|
+
// projects remain isolated going forward.
|
|
235
|
+
const globalLegacy = getGlobalFeedbackDir(options);
|
|
236
|
+
if (dirExists(globalLegacy)) return globalLegacy;
|
|
237
|
+
|
|
238
|
+
const projectDir = resolveProjectDir(options);
|
|
239
|
+
// New real projects select their collision-free local store before it is
|
|
240
|
+
// created, preventing the old first-write flip from global to local.
|
|
241
|
+
if (!isTransientProjectDir(projectDir, options)) return localThumbgate;
|
|
242
|
+
|
|
243
|
+
return globalLegacy;
|
|
225
244
|
}
|
|
226
245
|
|
|
227
246
|
function getFeedbackPaths(options = {}) {
|
|
@@ -154,6 +154,58 @@ function isGenericFeedbackText(value, signal) {
|
|
|
154
154
|
return rules.some((pattern) => pattern.test(normalized));
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Score a captured feedback entry with the reward rubric in judge-reward-function.
|
|
159
|
+
*
|
|
160
|
+
* `assessFeedbackActionability` answers a binary question — is this promotable at
|
|
161
|
+
* all. This answers the graded one: how good is the correction the operator wrote.
|
|
162
|
+
* A 👎 whose whatToChange is "be better" and a 👎 that says "run npm test before
|
|
163
|
+
* claiming green, see commit abc123" are both promotable; only one of them makes a
|
|
164
|
+
* useful prevention rule.
|
|
165
|
+
*
|
|
166
|
+
* Deterministic by construction. `buildCompositeReward` runs its Boolean rubric
|
|
167
|
+
* first and only consults an LLM judge if one is injected; we deliberately inject
|
|
168
|
+
* none, so this returns `scoringMode: 'deterministic_only'` and costs nothing. That
|
|
169
|
+
* matters: ANTHROPIC_API_KEY is frequently absent, and a scorer that silently
|
|
170
|
+
* degrades to nothing is worse than one that never claimed the LLM path.
|
|
171
|
+
*
|
|
172
|
+
* Reporting only — nothing here gates promotion. Wiring a fresh quality signal
|
|
173
|
+
* straight into enforcement would change which lessons become blocking rules, and
|
|
174
|
+
* that decision deserves its own change with its own evidence.
|
|
175
|
+
*
|
|
176
|
+
* @returns {{score: number, label: string, scoringMode: string, dimensions: object, passed: boolean}|null}
|
|
177
|
+
* null when there is no corrective text worth scoring.
|
|
178
|
+
*/
|
|
179
|
+
function scoreFeedbackReward(params = {}) {
|
|
180
|
+
const signal = normalizeFeedbackSignal(params.signal);
|
|
181
|
+
const prediction = normalizeFeedbackText(
|
|
182
|
+
signal === 'positive'
|
|
183
|
+
? (params.whatWorked || params.context)
|
|
184
|
+
: (params.whatToChange || params.whatWentWrong || params.context),
|
|
185
|
+
);
|
|
186
|
+
if (!prediction) return null;
|
|
187
|
+
|
|
188
|
+
// Lazy-require: judge-reward-function is a leaf, but feedback-quality is required
|
|
189
|
+
// by feedback-loop and three other modules — keep the import cost off that path
|
|
190
|
+
// until someone actually asks for a score.
|
|
191
|
+
let buildCompositeReward;
|
|
192
|
+
try {
|
|
193
|
+
({ buildCompositeReward } = require('./judge-reward-function'));
|
|
194
|
+
} catch {
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
if (typeof buildCompositeReward !== 'function') return null;
|
|
198
|
+
|
|
199
|
+
const reward = buildCompositeReward({ prediction });
|
|
200
|
+
return {
|
|
201
|
+
score: reward.score,
|
|
202
|
+
label: reward.label,
|
|
203
|
+
scoringMode: reward.scoringMode || 'deterministic_only',
|
|
204
|
+
dimensions: reward.deterministic ? reward.deterministic.dimensions : {},
|
|
205
|
+
passed: Boolean(reward.deterministic && reward.deterministic.passed),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
157
209
|
function assessFeedbackActionability(params = {}) {
|
|
158
210
|
const signal = normalizeFeedbackSignal(params.signal);
|
|
159
211
|
const primaryFields = signal === 'positive'
|
|
@@ -222,4 +274,5 @@ module.exports = {
|
|
|
222
274
|
isGenericFeedbackText,
|
|
223
275
|
assessFeedbackActionability,
|
|
224
276
|
buildClarificationMessage,
|
|
277
|
+
scoreFeedbackReward,
|
|
225
278
|
};
|
|
@@ -154,14 +154,19 @@ function scoreRecord(queryTokens, queryText, record) {
|
|
|
154
154
|
// ---------------------------------------------------------------------------
|
|
155
155
|
|
|
156
156
|
function searchFeedbackLog(queryText, limit = 5, options = {}) {
|
|
157
|
-
const
|
|
157
|
+
const feedbackDir = options.feedbackDir || getFeedbackDir();
|
|
158
|
+
const logPath = path.join(feedbackDir, 'feedback-log.jsonl');
|
|
158
159
|
let records = readJsonl(logPath);
|
|
159
160
|
|
|
160
|
-
// SQLite fallback
|
|
161
|
-
|
|
161
|
+
// SQLite fallback is allowed only inside the exact selected feedback root.
|
|
162
|
+
// Falling back through lesson-db's ambient default can cross project/tenant
|
|
163
|
+
// boundaries when a sparse project has zero or one JSONL row.
|
|
164
|
+
const lessonDbPath = path.join(feedbackDir, 'lessons.sqlite');
|
|
165
|
+
if (records.length <= 1 && fs.existsSync(lessonDbPath)) {
|
|
166
|
+
let db = null;
|
|
162
167
|
try {
|
|
163
168
|
const { initDB } = require('./lesson-db');
|
|
164
|
-
|
|
169
|
+
db = initDB(lessonDbPath);
|
|
165
170
|
const rows = db.prepare('SELECT * FROM lessons ORDER BY timestamp DESC LIMIT 500').all();
|
|
166
171
|
if (rows.length > records.length) {
|
|
167
172
|
records = rows.map((r) => ({
|
|
@@ -171,12 +176,17 @@ function searchFeedbackLog(queryText, limit = 5, options = {}) {
|
|
|
171
176
|
title: r.title || r.context,
|
|
172
177
|
tags: r.tags ? JSON.parse(r.tags) : [],
|
|
173
178
|
timestamp: r.timestamp,
|
|
174
|
-
whatWentWrong: r.what_went_wrong,
|
|
175
|
-
whatWorked: r.what_worked,
|
|
176
|
-
whatToChange: r.what_to_change,
|
|
179
|
+
whatWentWrong: r.whatWentWrong ?? r.what_went_wrong,
|
|
180
|
+
whatWorked: r.whatWorked ?? r.what_worked,
|
|
181
|
+
whatToChange: r.whatToChange ?? r.what_to_change,
|
|
177
182
|
}));
|
|
178
183
|
}
|
|
179
184
|
} catch { /* lesson-db not available */ }
|
|
185
|
+
finally {
|
|
186
|
+
if (db) {
|
|
187
|
+
try { db.close(); } catch { /* best-effort close */ }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
180
190
|
}
|
|
181
191
|
|
|
182
192
|
// Wildcard query: return all records sorted by recency
|
package/scripts/gates-engine.js
CHANGED
|
@@ -146,10 +146,18 @@ const SELF_PROTECT_HARD_FLOOR_GATE_IDS = new Set([
|
|
|
146
146
|
'self-protect-env-override',
|
|
147
147
|
'self-protect-hooks-disable',
|
|
148
148
|
]);
|
|
149
|
+
// An expired task-scope lease gets its OWN gate id so it can be exempted from the two downgrade
|
|
150
|
+
// paths without touching ordinary task-scope denials. Without this the fail-closed guarantee is
|
|
151
|
+
// cosmetic: applyEnforcementPosture turns denials into warnings by default, and applyDailyBlockCap
|
|
152
|
+
// does the same for capped free-tier users — so an edit under a lapsed lease would execute anyway.
|
|
153
|
+
// A lease that stops binding when you are busy or over quota is not a lease.
|
|
154
|
+
const TASK_SCOPE_LEASE_EXPIRED_GATE_ID = 'task-scope-lease-expired';
|
|
155
|
+
|
|
149
156
|
const UNCONDITIONAL_HARD_FLOOR_GATE_IDS = new Set([
|
|
150
157
|
'secret-exfiltration',
|
|
151
158
|
'security-vuln-scan',
|
|
152
159
|
'slopsquat-guard',
|
|
160
|
+
TASK_SCOPE_LEASE_EXPIRED_GATE_ID,
|
|
153
161
|
...SELF_PROTECT_HARD_FLOOR_GATE_IDS,
|
|
154
162
|
]);
|
|
155
163
|
// Issue #2782 (reported by Andy Martin, 2026-07-08): after the free-tier daily
|
|
@@ -161,6 +169,7 @@ const UNCONDITIONAL_HARD_FLOOR_GATE_IDS = new Set([
|
|
|
161
169
|
// map directly to CLAUDE.md's own hard-block list and must never be subject
|
|
162
170
|
// to the daily cap discount, regardless of tier or strict-mode setting.
|
|
163
171
|
const CATASTROPHIC_DECLARATIVE_GATE_IDS = new Set([
|
|
172
|
+
TASK_SCOPE_LEASE_EXPIRED_GATE_ID,
|
|
164
173
|
'force-push',
|
|
165
174
|
'git-reset-hard',
|
|
166
175
|
'git-clean-force',
|
|
@@ -425,6 +434,25 @@ function clampTtlMs(value, fallbackMs) {
|
|
|
425
434
|
return Math.min(Math.max(numeric, 60 * 1000), 24 * 60 * 60 * 1000);
|
|
426
435
|
}
|
|
427
436
|
|
|
437
|
+
// Default lease when a caller asks for one without saying how long. clampTtlMs floors at 60s.
|
|
438
|
+
const TASK_SCOPE_LEASE_MS = 15 * 60 * 1000;
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* A task scope with no `expiresAt` is permanent — that is the historical contract and every
|
|
442
|
+
* existing scope on disk has it. Only a scope that explicitly took a lease can expire.
|
|
443
|
+
*/
|
|
444
|
+
function isTaskScopeExpired(taskScope, nowMs = Date.now()) {
|
|
445
|
+
if (!taskScope || typeof taskScope !== 'object') return false;
|
|
446
|
+
// `expiresAt: null` means permanent, and it MUST be checked before the numeric coercion:
|
|
447
|
+
// Number(null) is 0, not NaN, so a null deadline would otherwise read as "expired in 1970".
|
|
448
|
+
// Combined with fail-closed enforcement that would revoke authority from every permanent
|
|
449
|
+
// scope the moment this shipped. Caught by tests/task-scope-lease.test.js.
|
|
450
|
+
if (taskScope.expiresAt == null) return false;
|
|
451
|
+
const deadline = Number(taskScope.expiresAt);
|
|
452
|
+
if (!Number.isFinite(deadline)) return false;
|
|
453
|
+
return nowMs >= deadline;
|
|
454
|
+
}
|
|
455
|
+
|
|
428
456
|
function loadGovernanceState() {
|
|
429
457
|
const raw = loadJSON(module.exports.GOVERNANCE_STATE_PATH);
|
|
430
458
|
const state = {
|
|
@@ -438,6 +466,12 @@ function loadGovernanceState() {
|
|
|
438
466
|
: null,
|
|
439
467
|
};
|
|
440
468
|
const now = Date.now();
|
|
469
|
+
// Annotate rather than delete. A vanished scope is indistinguishable from one never set, and
|
|
470
|
+
// the difference matters: "your lease lapsed, renew it" is a different instruction from
|
|
471
|
+
// "you never declared a scope".
|
|
472
|
+
if (state.taskScope) {
|
|
473
|
+
state.taskScope = { ...state.taskScope, expired: isTaskScopeExpired(state.taskScope, now) };
|
|
474
|
+
}
|
|
441
475
|
const activeApprovals = state.protectedApprovals.filter((entry) => {
|
|
442
476
|
if (!entry || typeof entry !== 'object') return false;
|
|
443
477
|
if (!entry.timestamp || !entry.expiresAt) return false;
|
|
@@ -485,6 +519,11 @@ function setTaskScope(scopeInput = {}) {
|
|
|
485
519
|
? scopeInput.protectedPaths
|
|
486
520
|
: DEFAULT_PROTECTED_FILE_GLOBS
|
|
487
521
|
), repoPath);
|
|
522
|
+
// Optional LEASE. Without ttlMs the scope is permanent, which is the historical behaviour
|
|
523
|
+
// and stays byte-identical. With ttlMs the scope becomes time-bounded authority: "write under
|
|
524
|
+
// ./src for 90 seconds" rather than a standing approval that never says when it stops.
|
|
525
|
+
const scopeNow = Date.now();
|
|
526
|
+
const leaseMs = scopeInput.ttlMs == null ? null : clampTtlMs(scopeInput.ttlMs, TASK_SCOPE_LEASE_MS);
|
|
488
527
|
const taskScope = {
|
|
489
528
|
taskId: String(scopeInput.taskId || '').trim() || null,
|
|
490
529
|
summary: String(scopeInput.summary || '').trim() || null,
|
|
@@ -493,7 +532,9 @@ function setTaskScope(scopeInput = {}) {
|
|
|
493
532
|
localOnly: scopeInput.localOnly === true,
|
|
494
533
|
repoPath,
|
|
495
534
|
createdAt: new Date().toISOString(),
|
|
496
|
-
timestamp:
|
|
535
|
+
timestamp: scopeNow,
|
|
536
|
+
leaseMs,
|
|
537
|
+
expiresAt: leaseMs == null ? null : scopeNow + leaseMs,
|
|
497
538
|
};
|
|
498
539
|
const state = loadGovernanceState();
|
|
499
540
|
state.taskScope = taskScope;
|
|
@@ -1195,11 +1236,28 @@ const COMMAND_WRAPPERS = new Set([
|
|
|
1195
1236
|
]);
|
|
1196
1237
|
const ENV_ASSIGNMENT_PREFIX = /^[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|[^\s]*)\s+/;
|
|
1197
1238
|
const WRAPPER_HEAD = /^([A-Za-z_][\w.-]*)\s+/;
|
|
1239
|
+
const LITERAL_COMMAND_SUBSTITUTION_HEADS = [
|
|
1240
|
+
/^\$\(\s*printf\s+(?:(?:['"]?%s['"]?)\s+)?(['"]?)([A-Za-z_][\w.-]*)\1\s*\)\s+/,
|
|
1241
|
+
/^\$\(\s*echo\s+(['"]?)([A-Za-z_][\w.-]*)\1\s*\)\s+/,
|
|
1242
|
+
/^\$\(\s*(?:command\s+-v|which)\s+(['"]?)([A-Za-z_][\w.-]*)\1\s*\)\s+/,
|
|
1243
|
+
];
|
|
1244
|
+
|
|
1245
|
+
function canonicalizeLiteralCommandSubstitutionHead(segment) {
|
|
1246
|
+
for (const pattern of LITERAL_COMMAND_SUBSTITUTION_HEADS) {
|
|
1247
|
+
const match = String(segment || '').match(pattern);
|
|
1248
|
+
if (match) return `${match[2]} ${String(segment).slice(match[0].length)}`;
|
|
1249
|
+
}
|
|
1250
|
+
return segment;
|
|
1251
|
+
}
|
|
1198
1252
|
|
|
1199
1253
|
function canonicalizeSegmentHead(segment) {
|
|
1200
1254
|
let text = String(segment || '').trim();
|
|
1201
1255
|
for (let i = 0; i < 12; i += 1) {
|
|
1202
1256
|
const before = text;
|
|
1257
|
+
// Resolve only literal, side-effect-free command-position substitutions.
|
|
1258
|
+
// Never execute or guess arbitrary shell; this closes common `$(printf git)`
|
|
1259
|
+
// and `$(command -v git)` spellings while keeping matching deterministic.
|
|
1260
|
+
text = canonicalizeLiteralCommandSubstitutionHead(text);
|
|
1203
1261
|
text = text.replace(ENV_ASSIGNMENT_PREFIX, '');
|
|
1204
1262
|
const wrapper = text.match(WRAPPER_HEAD);
|
|
1205
1263
|
if (wrapper && COMMAND_WRAPPERS.has(wrapper[1].toLowerCase())) {
|
|
@@ -1921,8 +1979,23 @@ function formatFileList(files, limit = 5) {
|
|
|
1921
1979
|
return `${items.slice(0, limit).join(', ')} (+${items.length - limit} more)`;
|
|
1922
1980
|
}
|
|
1923
1981
|
|
|
1924
|
-
function buildTaskScopeViolation(taskScope, affectedFiles) {
|
|
1982
|
+
function buildTaskScopeViolation(taskScope, affectedFiles, nowMs = Date.now()) {
|
|
1925
1983
|
if (!Array.isArray(affectedFiles) || affectedFiles.length === 0) return null;
|
|
1984
|
+
// EXPIRY FAILS CLOSED, and that direction is the whole point.
|
|
1985
|
+
//
|
|
1986
|
+
// A task scope is a restriction, so simply dropping it on expiry would make the agent MORE
|
|
1987
|
+
// powerful the moment its lease ran out — expiry would remove a boundary instead of removing
|
|
1988
|
+
// authority. A lease has to mean the opposite: while it is live you may work in these paths,
|
|
1989
|
+
// and when it lapses the authority is gone until it is renewed.
|
|
1990
|
+
if (taskScope && isTaskScopeExpired(taskScope, nowMs)) {
|
|
1991
|
+
return {
|
|
1992
|
+
reasonCode: 'expired_task_scope',
|
|
1993
|
+
outsideFiles: affectedFiles.slice(),
|
|
1994
|
+
allowedPaths: Array.isArray(taskScope.allowedPaths) ? taskScope.allowedPaths.slice() : [],
|
|
1995
|
+
summary: taskScope.summary || null,
|
|
1996
|
+
expiresAt: taskScope.expiresAt || null,
|
|
1997
|
+
};
|
|
1998
|
+
}
|
|
1926
1999
|
if (!taskScope || !Array.isArray(taskScope.allowedPaths) || taskScope.allowedPaths.length === 0) {
|
|
1927
2000
|
return {
|
|
1928
2001
|
reasonCode: 'missing_task_scope',
|
|
@@ -1995,6 +2068,11 @@ function buildBranchGovernanceViolation(governanceState, toolInput = {}, affecte
|
|
|
1995
2068
|
function buildGateMessage(gate, matchDetails) {
|
|
1996
2069
|
if (matchDetails && matchDetails.taskScopeViolation) {
|
|
1997
2070
|
const violation = matchDetails.taskScopeViolation;
|
|
2071
|
+
if (violation.reasonCode === 'expired_task_scope') {
|
|
2072
|
+
const lapsed = violation.expiresAt ? new Date(violation.expiresAt).toISOString() : 'unknown time';
|
|
2073
|
+
return `The task-scope lease expired at ${lapsed}, so its authority no longer applies. `
|
|
2074
|
+
+ `Renew it with set_task_scope (allowed paths were: ${formatFileList(violation.allowedPaths)}).`;
|
|
2075
|
+
}
|
|
1998
2076
|
if (violation.reasonCode === 'missing_task_scope') {
|
|
1999
2077
|
return `No task scope is declared for this high-risk action. Affected files: ${formatFileList(violation.outsideFiles)}.`;
|
|
2000
2078
|
}
|
|
@@ -2760,7 +2838,13 @@ async function evaluateGatesAsyncInner(toolName, toolInput, configPath) {
|
|
|
2760
2838
|
});
|
|
2761
2839
|
|
|
2762
2840
|
if (gate.action === 'block') {
|
|
2763
|
-
|
|
2841
|
+
// Expired leases report under their own gate id so neither the enforcement posture nor
|
|
2842
|
+
// the daily block cap can quietly turn this denial into a warning.
|
|
2843
|
+
const gateId = matchDetails && matchDetails.taskScopeViolation
|
|
2844
|
+
&& matchDetails.taskScopeViolation.reasonCode === 'expired_task_scope'
|
|
2845
|
+
? TASK_SCOPE_LEASE_EXPIRED_GATE_ID
|
|
2846
|
+
: gate.id;
|
|
2847
|
+
const denyResult = { decision: 'deny', gate: gateId, message, severity: gate.severity, reasoning };
|
|
2764
2848
|
// Free-tier daily block cap: after N blocks/day, deny → warn + upgrade CTA
|
|
2765
2849
|
const cappedResult = applyDailyBlockCap(denyResult);
|
|
2766
2850
|
if (cappedResult) {
|
|
@@ -2973,7 +3057,13 @@ function evaluateGatesInner(toolName, toolInput, configPath) {
|
|
|
2973
3057
|
const reasoning = buildReasoning(gate, toolName, toolInput, matchDetails);
|
|
2974
3058
|
|
|
2975
3059
|
if (gate.action === 'block') {
|
|
2976
|
-
|
|
3060
|
+
// Expired leases report under their own gate id so neither the enforcement posture nor
|
|
3061
|
+
// the daily block cap can quietly turn this denial into a warning.
|
|
3062
|
+
const gateId = matchDetails && matchDetails.taskScopeViolation
|
|
3063
|
+
&& matchDetails.taskScopeViolation.reasonCode === 'expired_task_scope'
|
|
3064
|
+
? TASK_SCOPE_LEASE_EXPIRED_GATE_ID
|
|
3065
|
+
: gate.id;
|
|
3066
|
+
const denyResult = { decision: 'deny', gate: gateId, message, severity: gate.severity, reasoning };
|
|
2977
3067
|
// Free-tier daily block cap: after N blocks/day, deny → warn + upgrade CTA
|
|
2978
3068
|
const cappedResult = applyDailyBlockCap(denyResult);
|
|
2979
3069
|
if (cappedResult) {
|
|
@@ -3968,6 +4058,10 @@ module.exports = {
|
|
|
3968
4058
|
loadGovernanceState,
|
|
3969
4059
|
saveGovernanceState,
|
|
3970
4060
|
setTaskScope,
|
|
4061
|
+
isTaskScopeExpired,
|
|
4062
|
+
TASK_SCOPE_LEASE_EXPIRED_GATE_ID,
|
|
4063
|
+
applyEnforcementPosture,
|
|
4064
|
+
buildTaskScopeViolation,
|
|
3971
4065
|
setBranchGovernance,
|
|
3972
4066
|
approveProtectedAction,
|
|
3973
4067
|
breakGlassEmergency,
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* generate-case-study-outreach.js — buyer outreach pack from dogfood case studies.
|
|
6
|
+
*
|
|
7
|
+
* No fabricated customer logos. Packs are built from public case-study anchors
|
|
8
|
+
* with first-party UTMs for the cash path.
|
|
9
|
+
*
|
|
10
|
+
* node scripts/generate-case-study-outreach.js --case=sudo-evasion
|
|
11
|
+
* node scripts/generate-case-study-outreach.js --case=sudo-evasion --json
|
|
12
|
+
* node scripts/generate-case-study-outreach.js --case=sudo-evasion --write
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const fs = require('node:fs');
|
|
16
|
+
const path = require('node:path');
|
|
17
|
+
|
|
18
|
+
const PROJECT_ROOT = path.resolve(__dirname, '..');
|
|
19
|
+
const DEFAULT_OUT_DIR = path.join(PROJECT_ROOT, 'docs', 'proof', 'outreach');
|
|
20
|
+
|
|
21
|
+
const CASES = Object.freeze({
|
|
22
|
+
'sudo-evasion': {
|
|
23
|
+
id: 'sudo-evasion',
|
|
24
|
+
title: 'A guardrail you could walk past with sudo',
|
|
25
|
+
anchor: 'sudo-evasion',
|
|
26
|
+
problem: 'Catastrophic PreToolUse gates matched the happy path but missed wrappers like `sudo rm -rf ~`.',
|
|
27
|
+
metric: '62 evasion holes → 0 on the published npm artifact',
|
|
28
|
+
result: 'Canonicalization + an adversarial grid (14 commands × 9 transforms) closed the class; CI + 6-hourly published-artifact jobs keep it closed.',
|
|
29
|
+
buyerPain: 'Your coding agent can re-spell a blocked command and walk past a regex denylist.',
|
|
30
|
+
ctaPrimary: 'diagnostic',
|
|
31
|
+
proofLinks: {
|
|
32
|
+
caseStudyPath: '/case-studies#sudo-evasion',
|
|
33
|
+
scorecardPath: '/eval-scorecard',
|
|
34
|
+
whitepaperPath: '/whitepaper',
|
|
35
|
+
diagnosticPath: '/diagnostic',
|
|
36
|
+
proPath: '/checkout/pro',
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
'fail-open': {
|
|
40
|
+
id: 'fail-open',
|
|
41
|
+
title: 'Production failure: a firewall enforcing nothing',
|
|
42
|
+
anchor: 'fail-open',
|
|
43
|
+
problem: 'A missing PreToolUse hook binary fails open — the product looked fine while blocking nothing.',
|
|
44
|
+
metric: 'Silent-gate canary + published-artifact deny checks',
|
|
45
|
+
result: 'Enforcement restored and verified with known-dangerous commands; silence is now treated as a P0 class.',
|
|
46
|
+
buyerPain: 'Green uptime does not mean your agent firewall is still firing.',
|
|
47
|
+
ctaPrimary: 'diagnostic',
|
|
48
|
+
proofLinks: {
|
|
49
|
+
caseStudyPath: '/case-studies#fail-open',
|
|
50
|
+
scorecardPath: '/eval-scorecard',
|
|
51
|
+
whitepaperPath: '/whitepaper',
|
|
52
|
+
diagnosticPath: '/diagnostic',
|
|
53
|
+
proPath: '/checkout/pro',
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
function parseArgs(argv = process.argv.slice(2)) {
|
|
59
|
+
const args = {
|
|
60
|
+
caseId: 'sudo-evasion',
|
|
61
|
+
json: false,
|
|
62
|
+
write: false,
|
|
63
|
+
help: false,
|
|
64
|
+
outDir: DEFAULT_OUT_DIR,
|
|
65
|
+
baseUrl: 'https://thumbgate.ai',
|
|
66
|
+
};
|
|
67
|
+
for (const arg of argv) {
|
|
68
|
+
if (arg === '--json') args.json = true;
|
|
69
|
+
else if (arg === '--write') args.write = true;
|
|
70
|
+
else if (arg === '--help' || arg === '-h') args.help = true;
|
|
71
|
+
else if (arg.startsWith('--case=')) args.caseId = arg.slice('--case='.length);
|
|
72
|
+
else if (arg.startsWith('--out-dir=')) args.outDir = path.resolve(arg.slice('--out-dir='.length));
|
|
73
|
+
else if (arg.startsWith('--base-url=')) args.baseUrl = arg.slice('--base-url='.length).replace(/\/$/, '');
|
|
74
|
+
}
|
|
75
|
+
return args;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function printHelp() {
|
|
79
|
+
console.log(`Usage: node scripts/generate-case-study-outreach.js --case=<id> [--write] [--json]
|
|
80
|
+
|
|
81
|
+
Cases: ${Object.keys(CASES).join(', ')}
|
|
82
|
+
`);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function withUtm(baseUrl, pathAndHash, campaign, content) {
|
|
86
|
+
const [pathname, hash = ''] = pathAndHash.split('#');
|
|
87
|
+
const url = new URL(pathname, baseUrl);
|
|
88
|
+
url.searchParams.set('utm_source', 'case_study_outreach');
|
|
89
|
+
url.searchParams.set('utm_medium', content);
|
|
90
|
+
url.searchParams.set('utm_campaign', campaign);
|
|
91
|
+
url.searchParams.set('cta_id', `${campaign}_${content}`);
|
|
92
|
+
const hashPart = hash ? `#${hash}` : '';
|
|
93
|
+
return `${url.toString()}${hashPart}`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function buildPack(caseDef, options = {}) {
|
|
97
|
+
const baseUrl = options.baseUrl || 'https://thumbgate.ai';
|
|
98
|
+
const campaign = `case_${caseDef.id.replace(/-/g, '_')}`;
|
|
99
|
+
const links = {
|
|
100
|
+
caseStudy: withUtm(baseUrl, caseDef.proofLinks.caseStudyPath, campaign, 'case_study'),
|
|
101
|
+
scorecard: withUtm(baseUrl, caseDef.proofLinks.scorecardPath, campaign, 'scorecard'),
|
|
102
|
+
whitepaper: withUtm(baseUrl, caseDef.proofLinks.whitepaperPath, campaign, 'whitepaper'),
|
|
103
|
+
diagnostic: withUtm(baseUrl, caseDef.proofLinks.diagnosticPath, campaign, 'diagnostic'),
|
|
104
|
+
pro: withUtm(baseUrl, `${caseDef.proofLinks.proPath}`, campaign, 'pro'),
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
const linkedin = [
|
|
108
|
+
caseDef.buyerPain,
|
|
109
|
+
'',
|
|
110
|
+
`We dogfooded this on ThumbGate itself: ${caseDef.metric}.`,
|
|
111
|
+
caseDef.result,
|
|
112
|
+
'',
|
|
113
|
+
`Full write-up (no fabricated logos): ${links.caseStudy}`,
|
|
114
|
+
`Live bench scorecard: ${links.scorecard}`,
|
|
115
|
+
'',
|
|
116
|
+
`If one repeated AI-agent failure is already costing you, the $499 Diagnostic installs one hard gate with regression proof: ${links.diagnostic}`,
|
|
117
|
+
].join('\n');
|
|
118
|
+
|
|
119
|
+
const email = [
|
|
120
|
+
`Subject: Your agent can walk past a regex denylist`,
|
|
121
|
+
'',
|
|
122
|
+
`Hi —`,
|
|
123
|
+
'',
|
|
124
|
+
caseDef.buyerPain,
|
|
125
|
+
'',
|
|
126
|
+
`Concrete proof from our own product loop (not a customer logo page):`,
|
|
127
|
+
`- ${caseDef.metric}`,
|
|
128
|
+
`- ${caseDef.result}`,
|
|
129
|
+
'',
|
|
130
|
+
`Case study: ${links.caseStudy}`,
|
|
131
|
+
`Scorecard: ${links.scorecard}`,
|
|
132
|
+
`White paper: ${links.whitepaper}`,
|
|
133
|
+
'',
|
|
134
|
+
`If you want this on one painful workflow this week: ${links.diagnostic}`,
|
|
135
|
+
`Self-serve Pro: ${links.pro}`,
|
|
136
|
+
'',
|
|
137
|
+
`— Igor`,
|
|
138
|
+
].join('\n');
|
|
139
|
+
|
|
140
|
+
const reddit = [
|
|
141
|
+
`**Problem:** ${caseDef.problem}`,
|
|
142
|
+
'',
|
|
143
|
+
`**What we measured:** ${caseDef.metric}`,
|
|
144
|
+
'',
|
|
145
|
+
`**What fixed it:** ${caseDef.result}`,
|
|
146
|
+
'',
|
|
147
|
+
`Public case study (dogfood, not a fake logo wall): ${links.caseStudy}`,
|
|
148
|
+
`Bench scorecard: ${links.scorecard}`,
|
|
149
|
+
].join('\n');
|
|
150
|
+
|
|
151
|
+
const markdown = [
|
|
152
|
+
`# Outreach pack — ${caseDef.title}`,
|
|
153
|
+
'',
|
|
154
|
+
`Case id: \`${caseDef.id}\``,
|
|
155
|
+
'',
|
|
156
|
+
'## Links (tracked)',
|
|
157
|
+
'',
|
|
158
|
+
`- Case study: ${links.caseStudy}`,
|
|
159
|
+
`- Scorecard: ${links.scorecard}`,
|
|
160
|
+
`- White paper: ${links.whitepaper}`,
|
|
161
|
+
`- Diagnostic $499: ${links.diagnostic}`,
|
|
162
|
+
`- Pro: ${links.pro}`,
|
|
163
|
+
'',
|
|
164
|
+
'## LinkedIn',
|
|
165
|
+
'',
|
|
166
|
+
linkedin,
|
|
167
|
+
'',
|
|
168
|
+
'## Email',
|
|
169
|
+
'',
|
|
170
|
+
'```',
|
|
171
|
+
email,
|
|
172
|
+
'```',
|
|
173
|
+
'',
|
|
174
|
+
'## Reddit / forum',
|
|
175
|
+
'',
|
|
176
|
+
reddit,
|
|
177
|
+
'',
|
|
178
|
+
'## Honesty',
|
|
179
|
+
'',
|
|
180
|
+
'First-party dogfood narrative only. Do not imply third-party customer endorsement.',
|
|
181
|
+
'',
|
|
182
|
+
].join('\n');
|
|
183
|
+
|
|
184
|
+
return {
|
|
185
|
+
caseId: caseDef.id,
|
|
186
|
+
title: caseDef.title,
|
|
187
|
+
links,
|
|
188
|
+
channels: {
|
|
189
|
+
linkedin,
|
|
190
|
+
email,
|
|
191
|
+
reddit,
|
|
192
|
+
},
|
|
193
|
+
markdown,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function generate(options = {}) {
|
|
198
|
+
const caseId = options.caseId || 'sudo-evasion';
|
|
199
|
+
const caseDef = CASES[caseId];
|
|
200
|
+
if (!caseDef) {
|
|
201
|
+
throw new Error(`Unknown case id: ${caseId}. Known: ${Object.keys(CASES).join(', ')}`);
|
|
202
|
+
}
|
|
203
|
+
const pack = buildPack(caseDef, { baseUrl: options.baseUrl });
|
|
204
|
+
let outPath = null;
|
|
205
|
+
if (options.write) {
|
|
206
|
+
const outDir = options.outDir || DEFAULT_OUT_DIR;
|
|
207
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
208
|
+
outPath = path.join(outDir, `case-study-outreach-${caseId}.md`);
|
|
209
|
+
fs.writeFileSync(outPath, pack.markdown, 'utf8');
|
|
210
|
+
}
|
|
211
|
+
return { ...pack, outPath };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function main(argv = process.argv.slice(2)) {
|
|
215
|
+
const args = parseArgs(argv);
|
|
216
|
+
if (args.help) {
|
|
217
|
+
printHelp();
|
|
218
|
+
return 0;
|
|
219
|
+
}
|
|
220
|
+
const result = generate(args);
|
|
221
|
+
if (args.json) {
|
|
222
|
+
console.log(JSON.stringify({
|
|
223
|
+
caseId: result.caseId,
|
|
224
|
+
title: result.title,
|
|
225
|
+
links: result.links,
|
|
226
|
+
channels: result.channels,
|
|
227
|
+
outPath: result.outPath,
|
|
228
|
+
}, null, 2));
|
|
229
|
+
} else if (result.outPath) {
|
|
230
|
+
console.log(`Wrote ${result.outPath}`);
|
|
231
|
+
} else {
|
|
232
|
+
console.log(result.markdown);
|
|
233
|
+
}
|
|
234
|
+
return 0;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (path.resolve(process.argv[1] || '') === path.resolve(__filename)) {
|
|
238
|
+
try {
|
|
239
|
+
process.exitCode = main();
|
|
240
|
+
} catch (err) {
|
|
241
|
+
console.error(err.message || err);
|
|
242
|
+
process.exitCode = 1;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
module.exports = {
|
|
247
|
+
CASES,
|
|
248
|
+
parseArgs,
|
|
249
|
+
withUtm,
|
|
250
|
+
buildPack,
|
|
251
|
+
generate,
|
|
252
|
+
main,
|
|
253
|
+
};
|