thumbgate 1.29.2 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +54 -16
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +143 -14
- package/adapters/opencode/opencode.json +1 -1
- package/bench/observability-eval-suite.json +2 -2
- package/bin/cli.js +154 -36
- package/config/evals/generation-quality-golden.json +95 -0
- package/config/evals/rag-answer-quality-golden.json +91 -0
- package/config/evals/retrieval-hybrid-ablation.json +66 -0
- package/config/evals/retrieval-ranking-golden.json +522 -0
- package/config/gates/default.json +217 -50
- package/config/mcp-allowlists.json +215 -185
- package/config/model-tiers.json +7 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/glama.json +6 -0
- package/package.json +94 -11
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +22 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/hero-thumbs.svg +68 -0
- package/public/assets/diagrams/loop.svg +40 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/self-improving-thumbs-loop.svg +105 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/compare.html +1 -0
- package/public/dashboard.html +126 -28
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +143 -13
- package/public/numbers.html +3 -2
- package/public/pricing.html +143 -30
- package/public/whitepaper.html +189 -0
- package/scripts/a-plus-evidence-scorecard.js +303 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/async-eval-observability.js +36 -11
- package/scripts/audit-trail.js +37 -1
- package/scripts/auto-promote-gates.js +149 -34
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/colbert-style-maxsim.js +236 -0
- package/scripts/cross-encoder-reranker.js +359 -126
- package/scripts/dashboard-chat.js +350 -17
- package/scripts/document-intake.js +283 -7
- package/scripts/eval-quality-suite.js +204 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +359 -189
- package/scripts/feedback-paths.js +32 -13
- package/scripts/feedback-quality.js +53 -0
- package/scripts/filesystem-search.js +17 -7
- package/scripts/gates-engine.js +98 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/harness-tool-names.js +70 -0
- package/scripts/hook-runtime.js +10 -3
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-db.js +16 -5
- package/scripts/lesson-embedding-index.js +67 -20
- package/scripts/lesson-embedding-maintenance.js +177 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-reranker.js +55 -9
- package/scripts/lesson-retrieval.js +375 -32
- package/scripts/lesson-search.js +48 -11
- package/scripts/llm-client.js +304 -15
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/model-tier-router.js +593 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/pragmatic-hybrid-search.js +379 -0
- package/scripts/published-cli.js +11 -1
- package/scripts/rag-document-pipeline.js +461 -0
- package/scripts/rag-structured-output.js +441 -0
- package/scripts/ragas-style-metrics.js +351 -0
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/request-envelope.js +178 -0
- package/scripts/rerank-pipeline.js +370 -0
- package/scripts/rerank-quality-eval.js +155 -0
- package/scripts/retrieval-hybrid-ablation.js +120 -0
- package/scripts/retrieval-quality-tier.js +118 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/secret-scanner.js +395 -4
- package/scripts/self-distill-agent.js +7 -1
- package/scripts/self-healing-check.js +25 -0
- package/scripts/skill-packs.js +183 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/statusline-links.js +1 -1
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/statusline.sh +8 -1
- package/scripts/telemetry-analytics.js +13 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/thumbgate-search.js +98 -6
- package/scripts/tier-budget-guard.js +186 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +95 -1
- package/scripts/vector-store.js +108 -4
- package/scripts/verify-marketing-pages-deployed.js +85 -3
- package/server.json +44 -0
- package/smithery.yaml +17 -0
- package/src/api/server.js +424 -99
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* generate-eval-scorecard.js — render public/eval-scorecard.html from ThumbGate Bench.
|
|
6
|
+
*
|
|
7
|
+
* Runs the committed golden suite (bench/thumbgate-bench.json) in an isolated
|
|
8
|
+
* runtime and publishes the measured rates so buyers can inspect tool-call
|
|
9
|
+
* correctness without cloning the repo. Regenerated via:
|
|
10
|
+
* npm run eval-scorecard:generate
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const fs = require('node:fs');
|
|
14
|
+
const path = require('node:path');
|
|
15
|
+
|
|
16
|
+
const PROJECT_ROOT = path.resolve(__dirname, '..');
|
|
17
|
+
const OUTPUT_PATH = path.join(PROJECT_ROOT, 'public', 'eval-scorecard.html');
|
|
18
|
+
const DEFAULT_SUITE = path.join(PROJECT_ROOT, 'bench', 'thumbgate-bench.json');
|
|
19
|
+
|
|
20
|
+
function escapeHtml(value) {
|
|
21
|
+
return String(value)
|
|
22
|
+
.replaceAll('&', '&')
|
|
23
|
+
.replaceAll('<', '<')
|
|
24
|
+
.replaceAll('>', '>')
|
|
25
|
+
.replaceAll('"', '"')
|
|
26
|
+
.replaceAll("'", ''');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function pct(rate) {
|
|
30
|
+
if (!Number.isFinite(rate)) return 'n/a';
|
|
31
|
+
return `${(rate * 100).toFixed(1)}%`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function loadVersion() {
|
|
35
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(PROJECT_ROOT, 'package.json'), 'utf8'));
|
|
36
|
+
return pkg.version || '0.0.0';
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function runBench() {
|
|
40
|
+
// Prefer the public API surface of thumbgate-bench.
|
|
41
|
+
const bench = require('./thumbgate-bench');
|
|
42
|
+
if (typeof bench.runBenchmark === 'function') {
|
|
43
|
+
return bench.runBenchmark({
|
|
44
|
+
suitePath: DEFAULT_SUITE,
|
|
45
|
+
minScore: 90,
|
|
46
|
+
useRuntimeState: false,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
if (typeof bench.main === 'function') {
|
|
50
|
+
// Some builds only export CLI entry; fall back to subprocess below.
|
|
51
|
+
}
|
|
52
|
+
const { spawnSync } = require('node:child_process');
|
|
53
|
+
const result = spawnSync(
|
|
54
|
+
process.execPath,
|
|
55
|
+
[path.join(PROJECT_ROOT, 'scripts', 'thumbgate-bench.js'), '--json'],
|
|
56
|
+
{ cwd: PROJECT_ROOT, encoding: 'utf8', maxBuffer: 8 * 1024 * 1024 },
|
|
57
|
+
);
|
|
58
|
+
if (result.status !== 0 && !result.stdout) {
|
|
59
|
+
throw new Error(result.stderr || 'thumbgate-bench failed');
|
|
60
|
+
}
|
|
61
|
+
const text = String(result.stdout || '').trim();
|
|
62
|
+
const start = text.indexOf('{');
|
|
63
|
+
const end = text.lastIndexOf('}');
|
|
64
|
+
if (start < 0 || end < start) {
|
|
65
|
+
throw new Error('thumbgate-bench did not emit JSON');
|
|
66
|
+
}
|
|
67
|
+
return JSON.parse(text.slice(start, end + 1));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function renderScorecard(input) {
|
|
71
|
+
const {
|
|
72
|
+
version,
|
|
73
|
+
nowIso,
|
|
74
|
+
nowDate,
|
|
75
|
+
metrics,
|
|
76
|
+
passed,
|
|
77
|
+
scenarios,
|
|
78
|
+
sourcePath,
|
|
79
|
+
} = input;
|
|
80
|
+
|
|
81
|
+
const m = metrics || {};
|
|
82
|
+
const rows = (scenarios || []).map((s) => {
|
|
83
|
+
const status = s.passed
|
|
84
|
+
? '<span class="good">PASS</span>'
|
|
85
|
+
: '<span class="bad">FAIL</span>';
|
|
86
|
+
return `<tr>
|
|
87
|
+
<td><code>${escapeHtml(s.id)}</code></td>
|
|
88
|
+
<td>${escapeHtml(s.service || '')}</td>
|
|
89
|
+
<td>${s.unsafe ? 'unsafe' : 'safe'}</td>
|
|
90
|
+
<td><code>${escapeHtml(s.expectedDecision)}</code></td>
|
|
91
|
+
<td><code>${escapeHtml(s.actualDecision)}</code></td>
|
|
92
|
+
<td>${status}</td>
|
|
93
|
+
</tr>`;
|
|
94
|
+
}).join('\n');
|
|
95
|
+
|
|
96
|
+
const softwareLd = {
|
|
97
|
+
'@context': 'https://schema.org',
|
|
98
|
+
'@type': 'Dataset',
|
|
99
|
+
name: 'ThumbGate Bench Scorecard',
|
|
100
|
+
description:
|
|
101
|
+
'Deterministic pre-action gate benchmark metrics: task success, unsafe-action rate, capability rate, false-block rate, and replay stability.',
|
|
102
|
+
url: 'https://thumbgate.ai/eval-scorecard',
|
|
103
|
+
dateModified: nowDate,
|
|
104
|
+
creator: {
|
|
105
|
+
'@type': 'Person',
|
|
106
|
+
name: 'Igor Ganapolsky',
|
|
107
|
+
url: 'https://github.com/IgorGanapolsky',
|
|
108
|
+
},
|
|
109
|
+
variableMeasured: [
|
|
110
|
+
{ '@type': 'PropertyValue', name: 'score', value: m.score },
|
|
111
|
+
{ '@type': 'PropertyValue', name: 'taskSuccessRate', value: m.taskSuccessRate },
|
|
112
|
+
{ '@type': 'PropertyValue', name: 'unsafeActionRate', value: m.unsafeActionRate },
|
|
113
|
+
{ '@type': 'PropertyValue', name: 'blockedUnsafeRate', value: m.blockedUnsafeRate },
|
|
114
|
+
{ '@type': 'PropertyValue', name: 'capabilityRate', value: m.capabilityRate },
|
|
115
|
+
{ '@type': 'PropertyValue', name: 'falseBlockRate', value: m.falseBlockRate },
|
|
116
|
+
{ '@type': 'PropertyValue', name: 'replayStability', value: m.replayStability },
|
|
117
|
+
],
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
return `<!DOCTYPE html>
|
|
121
|
+
<html lang="en">
|
|
122
|
+
<head>
|
|
123
|
+
<meta charset="UTF-8">
|
|
124
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
125
|
+
<meta name="generator" content="ThumbGate">
|
|
126
|
+
<meta name="author" content="Igor Ganapolsky">
|
|
127
|
+
<title>ThumbGate — Eval Scorecard | ThumbGate Bench Metrics</title>
|
|
128
|
+
<meta name="description" content="Live-regenerated ThumbGate Bench scorecard: task success, unsafe-action rate (must be 0), capability rate, false-block rate, and per-scenario tool-call decisions.">
|
|
129
|
+
<meta property="og:title" content="ThumbGate — Eval Scorecard">
|
|
130
|
+
<meta property="og:description" content="Deterministic gate benchmark metrics buyers can re-run: unsafeActionRate must stay 0.">
|
|
131
|
+
<meta property="og:type" content="website">
|
|
132
|
+
<meta property="og:url" content="https://thumbgate.ai/eval-scorecard">
|
|
133
|
+
<link rel="canonical" href="https://thumbgate.ai/eval-scorecard">
|
|
134
|
+
<link rel="icon" type="image/png" href="/thumbgate-icon.png">
|
|
135
|
+
<script defer data-domain="thumbgate.ai" src="https://plausible.io/js/script.js"></script>
|
|
136
|
+
<script type="application/ld+json">${JSON.stringify(softwareLd)}</script>
|
|
137
|
+
<style>
|
|
138
|
+
:root {
|
|
139
|
+
--bg:#0b0f14; --panel:#111823; --border:#1e2a3a; --text:#e6edf3;
|
|
140
|
+
--muted:#8b98a5; --cyan:#39c5cf; --green:#3fb950; --red:#f85149; --amber:#d29922;
|
|
141
|
+
}
|
|
142
|
+
* { margin:0; padding:0; box-sizing:border-box; }
|
|
143
|
+
body { background:var(--bg); color:var(--text); font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif; line-height:1.6; }
|
|
144
|
+
nav { padding:1rem 2rem; border-bottom:1px solid var(--border); display:flex; gap:1.25rem; flex-wrap:wrap; align-items:center; }
|
|
145
|
+
nav a { color:var(--muted); text-decoration:none; font-size:0.9rem; }
|
|
146
|
+
nav a:hover { color:var(--cyan); }
|
|
147
|
+
nav .brand { color:var(--text); font-weight:700; }
|
|
148
|
+
.container { max-width:960px; margin:0 auto; padding:2.5rem 1.5rem 4rem; }
|
|
149
|
+
h1 { font-size:2rem; margin-bottom:0.4rem; }
|
|
150
|
+
h2 { font-size:1.3rem; margin:2.2rem 0 0.75rem; color:var(--cyan); }
|
|
151
|
+
.subtitle { color:var(--muted); font-size:1.05rem; margin-bottom:1.25rem; }
|
|
152
|
+
.grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:0.75rem; margin:1.25rem 0; }
|
|
153
|
+
.metric { background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:1rem; }
|
|
154
|
+
.metric .label { color:var(--muted); font-size:0.78rem; text-transform:uppercase; letter-spacing:0.04em; }
|
|
155
|
+
.metric .value { font-size:1.55rem; font-weight:700; margin-top:0.25rem; }
|
|
156
|
+
.good { color:var(--green); } .bad { color:var(--red); } .warn { color:var(--amber); }
|
|
157
|
+
table { width:100%; border-collapse:collapse; margin:1rem 0; font-size:0.9rem; }
|
|
158
|
+
th, td { text-align:left; padding:0.5rem 0.6rem; border-bottom:1px solid var(--border); vertical-align:top; }
|
|
159
|
+
th { color:var(--muted); font-weight:600; }
|
|
160
|
+
code { background:#0d1420; border:1px solid var(--border); border-radius:4px; padding:0.1rem 0.35rem; font-size:0.84em; color:var(--cyan); }
|
|
161
|
+
pre { background:#0d1420; border:1px solid var(--border); border-radius:8px; padding:1rem; overflow-x:auto; font-size:0.85rem; margin:1rem 0; }
|
|
162
|
+
.panel { background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:1.1rem 1.25rem; margin:1rem 0; }
|
|
163
|
+
.footer-note { color:var(--muted); font-size:0.85rem; margin-top:2.5rem; text-align:center; }
|
|
164
|
+
.footer-note a { color:var(--cyan); text-decoration:none; }
|
|
165
|
+
a.cta { display:inline-block; background:var(--cyan); color:#001018; font-weight:700; padding:0.7rem 1rem; border-radius:8px; text-decoration:none; margin-right:0.75rem; }
|
|
166
|
+
a.secondary { color:var(--cyan); }
|
|
167
|
+
</style>
|
|
168
|
+
</head>
|
|
169
|
+
<body>
|
|
170
|
+
<nav>
|
|
171
|
+
<a class="brand" href="/">ThumbGate</a>
|
|
172
|
+
<a href="/whitepaper">White paper</a>
|
|
173
|
+
<a href="/architecture">Architecture</a>
|
|
174
|
+
<a href="/eval-scorecard">Scorecard</a>
|
|
175
|
+
<a href="/evaluations">Evaluations</a>
|
|
176
|
+
<a href="/case-studies">Case studies</a>
|
|
177
|
+
<a href="/numbers">Numbers</a>
|
|
178
|
+
<a href="/pricing">Pricing</a>
|
|
179
|
+
</nav>
|
|
180
|
+
<main class="container">
|
|
181
|
+
<h1>Eval scorecard</h1>
|
|
182
|
+
<p class="subtitle">Regenerated ThumbGate Bench results for version <strong>${escapeHtml(version)}</strong>. Updated: ${escapeHtml(nowDate)}. Suite: <code>${escapeHtml(sourcePath || 'bench/thumbgate-bench.json')}</code>.</p>
|
|
183
|
+
|
|
184
|
+
<div class="panel">
|
|
185
|
+
<strong>Pass criteria:</strong>
|
|
186
|
+
<code>unsafeActionRate</code> must be <span class="good">0</span>,
|
|
187
|
+
score ≥ 90, safe work still allowed (<code>capabilityRate</code>),
|
|
188
|
+
and decisions must replay stably.
|
|
189
|
+
Overall: ${passed ? '<span class="good">PASSED</span>' : '<span class="bad">FAILED</span>'} · composite score <strong>${escapeHtml(String(m.score ?? 'n/a'))}</strong>
|
|
190
|
+
<br><br>
|
|
191
|
+
<strong>Reproducibility:</strong> the generator runs ThumbGate Bench in an
|
|
192
|
+
<em>isolated</em> runtime with <code>THUMBGATE_STRICT_ENFORCEMENT=1</code>
|
|
193
|
+
so golden <code>deny</code> expectations are not downgraded by warn-by-default
|
|
194
|
+
posture or free-tier daily-cap state from the operator machine.
|
|
195
|
+
</div>
|
|
196
|
+
|
|
197
|
+
<div class="grid">
|
|
198
|
+
<div class="metric"><div class="label">Task success</div><div class="value good">${escapeHtml(pct(m.taskSuccessRate))}</div></div>
|
|
199
|
+
<div class="metric"><div class="label">Unsafe allowed</div><div class="value ${m.unsafeActionRate === 0 ? 'good' : 'bad'}">${escapeHtml(pct(m.unsafeActionRate))}</div></div>
|
|
200
|
+
<div class="metric"><div class="label">Unsafe blocked</div><div class="value">${escapeHtml(pct(m.blockedUnsafeRate))}</div></div>
|
|
201
|
+
<div class="metric"><div class="label">Capability</div><div class="value">${escapeHtml(pct(m.capabilityRate))}</div></div>
|
|
202
|
+
<div class="metric"><div class="label">False blocks</div><div class="value ${m.falseBlockRate === 0 ? 'good' : 'warn'}">${escapeHtml(pct(m.falseBlockRate))}</div></div>
|
|
203
|
+
<div class="metric"><div class="label">Replay stability</div><div class="value">${escapeHtml(pct(m.replayStability))}</div></div>
|
|
204
|
+
</div>
|
|
205
|
+
|
|
206
|
+
<h2>Per-scenario tool-call decisions</h2>
|
|
207
|
+
<p>Each row is a golden tool-call scenario: expected decision vs actual PreToolUse decision.</p>
|
|
208
|
+
<table>
|
|
209
|
+
<thead>
|
|
210
|
+
<tr><th>Scenario</th><th>Service</th><th>Class</th><th>Expected</th><th>Actual</th><th>Result</th></tr>
|
|
211
|
+
</thead>
|
|
212
|
+
<tbody>
|
|
213
|
+
${rows}
|
|
214
|
+
</tbody>
|
|
215
|
+
</table>
|
|
216
|
+
|
|
217
|
+
<h2>Reproduce locally</h2>
|
|
218
|
+
<pre>git clone https://github.com/IgorGanapolsky/ThumbGate
|
|
219
|
+
cd ThumbGate && npm ci
|
|
220
|
+
npm run thumbgate:bench -- --json
|
|
221
|
+
npm run eval-scorecard:generate</pre>
|
|
222
|
+
|
|
223
|
+
<p>
|
|
224
|
+
<a class="cta" href="/whitepaper">Read the evaluation white paper</a>
|
|
225
|
+
<a class="secondary" href="https://github.com/IgorGanapolsky/ThumbGate/blob/main/docs/THUMBGATE_BENCH.md">Bench methodology on GitHub →</a>
|
|
226
|
+
</p>
|
|
227
|
+
|
|
228
|
+
<p class="footer-note">
|
|
229
|
+
Generated at ${escapeHtml(nowIso)}. First-party measurement only — not customer traction.
|
|
230
|
+
Related: <a href="/evaluations">ML evaluations</a> · <a href="/architecture">Architecture diagrams</a> ·
|
|
231
|
+
<a href="https://github.com/IgorGanapolsky/ThumbGate/blob/main/docs/VERIFICATION_EVIDENCE.md">Verification evidence</a>
|
|
232
|
+
</p>
|
|
233
|
+
</main>
|
|
234
|
+
</body>
|
|
235
|
+
</html>
|
|
236
|
+
`;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function generate(options = {}) {
|
|
240
|
+
const now = options.now instanceof Date ? options.now : new Date();
|
|
241
|
+
const nowIso = now.toISOString();
|
|
242
|
+
const nowDate = nowIso.slice(0, 10);
|
|
243
|
+
const version = options.version || loadVersion();
|
|
244
|
+
const report = options.report || runBench();
|
|
245
|
+
const html = renderScorecard({
|
|
246
|
+
version,
|
|
247
|
+
nowIso,
|
|
248
|
+
nowDate,
|
|
249
|
+
metrics: report.metrics || report,
|
|
250
|
+
passed: report.passed !== false,
|
|
251
|
+
scenarios: report.scenarios || [],
|
|
252
|
+
sourcePath: report.sourcePath || 'bench/thumbgate-bench.json',
|
|
253
|
+
});
|
|
254
|
+
const outPath = options.outputPath || OUTPUT_PATH;
|
|
255
|
+
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
|
256
|
+
fs.writeFileSync(outPath, html, 'utf8');
|
|
257
|
+
return { outPath, html, report };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
if (path.resolve(process.argv[1] || '') === path.resolve(__filename)) {
|
|
261
|
+
try {
|
|
262
|
+
const { outPath, report } = generate();
|
|
263
|
+
const score = report.metrics?.score ?? report.score;
|
|
264
|
+
console.log(`Wrote ${outPath} (score=${score}, passed=${report.passed !== false})`);
|
|
265
|
+
} catch (err) {
|
|
266
|
+
console.error(err.message || err);
|
|
267
|
+
process.exit(1);
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
module.exports = {
|
|
272
|
+
generate,
|
|
273
|
+
renderScorecard,
|
|
274
|
+
runBench,
|
|
275
|
+
OUTPUT_PATH,
|
|
276
|
+
};
|
|
@@ -1,6 +1,183 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
|
+
const MARKETING_AGENT_CAMPAIGN_ID = 'marketing_agent_governance_20260727';
|
|
5
|
+
const CAMPAIGN_ALIASES = Object.freeze({
|
|
6
|
+
mg27: MARKETING_AGENT_CAMPAIGN_ID,
|
|
7
|
+
});
|
|
8
|
+
const CAMPAIGN_BUYER_ORIGIN = 'https://thumbgate-production.up.railway.app';
|
|
9
|
+
|
|
10
|
+
function campaignChannel(
|
|
11
|
+
channel,
|
|
12
|
+
permalink,
|
|
13
|
+
medium,
|
|
14
|
+
content = null,
|
|
15
|
+
campaignId = MARKETING_AGENT_CAMPAIGN_ID
|
|
16
|
+
) {
|
|
17
|
+
const buyerUrl = new URL('/go/pro', CAMPAIGN_BUYER_ORIGIN);
|
|
18
|
+
buyerUrl.searchParams.set('utm_source', channel);
|
|
19
|
+
buyerUrl.searchParams.set('utm_medium', medium);
|
|
20
|
+
buyerUrl.searchParams.set('utm_campaign', campaignId);
|
|
21
|
+
if (content) buyerUrl.searchParams.set('utm_content', content);
|
|
22
|
+
return Object.freeze({
|
|
23
|
+
channel,
|
|
24
|
+
status: 'LIVE',
|
|
25
|
+
permalink,
|
|
26
|
+
trackedBuyerUrl: buyerUrl.toString(),
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const MARKETING_AGENT_CAMPAIGN = Object.freeze({
|
|
31
|
+
campaignId: MARKETING_AGENT_CAMPAIGN_ID,
|
|
32
|
+
aliases: ['mg27'],
|
|
33
|
+
episode: {
|
|
34
|
+
title: 'Marketing Agents Are Too Good Now',
|
|
35
|
+
url: 'https://www.youtube.com/watch?v=U2hogriGmEw',
|
|
36
|
+
},
|
|
37
|
+
channels: [
|
|
38
|
+
campaignChannel(
|
|
39
|
+
'linkedin',
|
|
40
|
+
'https://www.linkedin.com/feed/update/urn:li:share:7487654549785128960/',
|
|
41
|
+
'organic_social',
|
|
42
|
+
'episode_response'
|
|
43
|
+
),
|
|
44
|
+
campaignChannel(
|
|
45
|
+
'hashnode',
|
|
46
|
+
'https://ai-agent-blog-12345.hashnode.dev/your-marketing-agent-can-publish-and-pause-ads-who-gates-the-write',
|
|
47
|
+
'organic_article',
|
|
48
|
+
'episode_deep_dive'
|
|
49
|
+
),
|
|
50
|
+
campaignChannel(
|
|
51
|
+
'bluesky',
|
|
52
|
+
'https://bsky.app/profile/iganapolsky.bsky.social/post/3mro3mkmrzc2y',
|
|
53
|
+
'social',
|
|
54
|
+
null,
|
|
55
|
+
'mg27'
|
|
56
|
+
),
|
|
57
|
+
campaignChannel(
|
|
58
|
+
'threads',
|
|
59
|
+
'https://www.threads.com/@igorganapolsky/post/DbUMBzXDlj8',
|
|
60
|
+
'social',
|
|
61
|
+
null,
|
|
62
|
+
'mg27'
|
|
63
|
+
),
|
|
64
|
+
campaignChannel(
|
|
65
|
+
'instagram',
|
|
66
|
+
'https://www.instagram.com/igorganapolsky/p/DbUNjFsDQz3/',
|
|
67
|
+
'organic_social',
|
|
68
|
+
'episode_card'
|
|
69
|
+
),
|
|
70
|
+
campaignChannel(
|
|
71
|
+
'reddit',
|
|
72
|
+
'https://www.reddit.com/r/SideProject/comments/1v8i0it/i_built_a_preaction_firewall_for_ai_agents_that/',
|
|
73
|
+
'organic_social',
|
|
74
|
+
'sideproject_build'
|
|
75
|
+
),
|
|
76
|
+
campaignChannel(
|
|
77
|
+
'youtube',
|
|
78
|
+
'https://www.youtube.com/post/UgkxERIbGUvSgCkGQ_dx2W0nbTl5_abcF17O',
|
|
79
|
+
'community_post',
|
|
80
|
+
'episode_response'
|
|
81
|
+
),
|
|
82
|
+
],
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
function normalizeCampaignId(value) {
|
|
86
|
+
const normalized = String(value || '').trim();
|
|
87
|
+
if (!normalized) return null;
|
|
88
|
+
return CAMPAIGN_ALIASES[normalized] || normalized;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function campaignAttributionKeys(campaign = MARKETING_AGENT_CAMPAIGN) {
|
|
92
|
+
return [...new Set([
|
|
93
|
+
campaign.campaignId,
|
|
94
|
+
...(campaign.aliases || []),
|
|
95
|
+
].map((value) => String(value || '').trim()).filter(Boolean))];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function validateCampaignChannel(entry, campaign, seenChannels, seenSources) {
|
|
99
|
+
const issues = [];
|
|
100
|
+
const channel = String(entry.channel || '').trim().toLowerCase();
|
|
101
|
+
if (!channel || seenChannels.has(channel)) {
|
|
102
|
+
issues.push(`duplicate_or_missing_channel:${channel || 'unknown'}`);
|
|
103
|
+
}
|
|
104
|
+
seenChannels.add(channel);
|
|
105
|
+
|
|
106
|
+
if (entry.status !== 'LIVE') {
|
|
107
|
+
issues.push(`channel_not_live:${channel || 'unknown'}`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
let permalink;
|
|
111
|
+
let trackedBuyerUrl;
|
|
112
|
+
try {
|
|
113
|
+
permalink = new URL(entry.permalink);
|
|
114
|
+
trackedBuyerUrl = new URL(entry.trackedBuyerUrl);
|
|
115
|
+
} catch {
|
|
116
|
+
issues.push(`invalid_url:${channel || 'unknown'}`);
|
|
117
|
+
return issues;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (permalink.protocol !== 'https:') {
|
|
121
|
+
issues.push(`permalink_not_https:${channel}`);
|
|
122
|
+
}
|
|
123
|
+
if (
|
|
124
|
+
trackedBuyerUrl.protocol !== 'https:'
|
|
125
|
+
|| trackedBuyerUrl.origin !== CAMPAIGN_BUYER_ORIGIN
|
|
126
|
+
|| trackedBuyerUrl.pathname !== '/go/pro'
|
|
127
|
+
) {
|
|
128
|
+
issues.push(`buyer_path_not_canonical:${channel}`);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const source = trackedBuyerUrl.searchParams.get('utm_source');
|
|
132
|
+
if (source !== channel || seenSources.has(source)) {
|
|
133
|
+
issues.push(`source_mismatch_or_duplicate:${channel}`);
|
|
134
|
+
}
|
|
135
|
+
seenSources.add(source);
|
|
136
|
+
|
|
137
|
+
if (
|
|
138
|
+
normalizeCampaignId(trackedBuyerUrl.searchParams.get('utm_campaign'))
|
|
139
|
+
!== campaign.campaignId
|
|
140
|
+
) {
|
|
141
|
+
issues.push(`campaign_mismatch:${channel}`);
|
|
142
|
+
}
|
|
143
|
+
if (!trackedBuyerUrl.searchParams.get('utm_medium')) {
|
|
144
|
+
issues.push(`missing_medium:${channel}`);
|
|
145
|
+
}
|
|
146
|
+
return issues;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function validateMarketingAgentCampaign(campaign = MARKETING_AGENT_CAMPAIGN) {
|
|
150
|
+
const issues = [];
|
|
151
|
+
const channels = Array.isArray(campaign.channels) ? campaign.channels : [];
|
|
152
|
+
const seenChannels = new Set();
|
|
153
|
+
const seenSources = new Set();
|
|
154
|
+
|
|
155
|
+
if (normalizeCampaignId(campaign.campaignId) !== MARKETING_AGENT_CAMPAIGN_ID) {
|
|
156
|
+
issues.push('campaign_id_must_be_canonical');
|
|
157
|
+
}
|
|
158
|
+
if (campaign.episode?.url !== 'https://www.youtube.com/watch?v=U2hogriGmEw') {
|
|
159
|
+
issues.push('episode_url_mismatch');
|
|
160
|
+
}
|
|
161
|
+
if (channels.length !== 7) {
|
|
162
|
+
issues.push('expected_seven_channels');
|
|
163
|
+
}
|
|
164
|
+
for (const entry of channels) {
|
|
165
|
+
issues.push(...validateCampaignChannel(
|
|
166
|
+
entry,
|
|
167
|
+
campaign,
|
|
168
|
+
seenChannels,
|
|
169
|
+
seenSources
|
|
170
|
+
));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return {
|
|
174
|
+
ok: issues.length === 0,
|
|
175
|
+
issues,
|
|
176
|
+
channelCount: channels.length,
|
|
177
|
+
campaignId: campaign.campaignId,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
4
181
|
function buildCreatorGrowthCampaign(input = {}) {
|
|
5
182
|
const appUrl = input.appUrl || 'https://thumbgate-production.up.railway.app';
|
|
6
183
|
const webinarTitle = input.webinarTitle || 'Stop AI Agents From Repeating Expensive Mistakes';
|
|
@@ -45,5 +222,11 @@ function buildCreatorGrowthCampaign(input = {}) {
|
|
|
45
222
|
}
|
|
46
223
|
|
|
47
224
|
module.exports = {
|
|
225
|
+
CAMPAIGN_ALIASES,
|
|
226
|
+
MARKETING_AGENT_CAMPAIGN,
|
|
227
|
+
MARKETING_AGENT_CAMPAIGN_ID,
|
|
48
228
|
buildCreatorGrowthCampaign,
|
|
229
|
+
campaignAttributionKeys,
|
|
230
|
+
normalizeCampaignId,
|
|
231
|
+
validateMarketingAgentCampaign,
|
|
49
232
|
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Harness tool names -> canonical gate-engine names.
|
|
5
|
+
*
|
|
6
|
+
* Policy gates match Claude Code's vocabulary (Bash, Write, Edit, MultiEdit). Other
|
|
7
|
+
* harnesses use their own: Cline's .clinerules instructs the agent to gate-check
|
|
8
|
+
* `execute_command`, `write_to_file`, `replace_in_file` and `browser_action`.
|
|
9
|
+
* Forwarding those through unchanged meant every gate silently missed — a strict
|
|
10
|
+
* `gate_check` on `{tool_name: "execute_command", command: "rm -rf /"}` returned ALLOW,
|
|
11
|
+
* which is precisely the scenario the adapter advertises as protected.
|
|
12
|
+
*
|
|
13
|
+
* Argument keys differ too: Cline writes use `path`, Claude Code uses `file_path`.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const TOOL_NAME_MAP = new Map(Object.entries({
|
|
17
|
+
// Cline / Roo Code
|
|
18
|
+
execute_command: 'Bash',
|
|
19
|
+
write_to_file: 'Write',
|
|
20
|
+
replace_in_file: 'Edit',
|
|
21
|
+
apply_diff: 'Edit',
|
|
22
|
+
insert_content: 'Edit',
|
|
23
|
+
search_and_replace: 'Edit',
|
|
24
|
+
// OpenCode / generic MCP
|
|
25
|
+
bash: 'Bash',
|
|
26
|
+
shell: 'Bash',
|
|
27
|
+
run_command: 'Bash',
|
|
28
|
+
edit_file: 'Edit',
|
|
29
|
+
write_file: 'Write',
|
|
30
|
+
create_file: 'Write',
|
|
31
|
+
str_replace_editor: 'Edit',
|
|
32
|
+
// Cursor
|
|
33
|
+
run_terminal_cmd: 'Bash',
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
const ARG_KEY_MAP = new Map(Object.entries({
|
|
37
|
+
path: 'file_path',
|
|
38
|
+
filePath: 'file_path',
|
|
39
|
+
target_file: 'file_path',
|
|
40
|
+
cmd: 'command',
|
|
41
|
+
shell_command: 'command',
|
|
42
|
+
diff: 'new_string',
|
|
43
|
+
}));
|
|
44
|
+
|
|
45
|
+
function canonicalizeToolName(toolName) {
|
|
46
|
+
const raw = String(toolName || '').trim();
|
|
47
|
+
if (!raw) return raw;
|
|
48
|
+
return TOOL_NAME_MAP.get(raw) || TOOL_NAME_MAP.get(raw.toLowerCase()) || raw;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function canonicalizeToolInput(toolInput) {
|
|
52
|
+
if (!toolInput || typeof toolInput !== 'object' || Array.isArray(toolInput)) return toolInput || {};
|
|
53
|
+
const out = {};
|
|
54
|
+
for (const [key, value] of Object.entries(toolInput)) {
|
|
55
|
+
const canonicalKey = ARG_KEY_MAP.get(key) || key;
|
|
56
|
+
// Never let a mapped key clobber one the caller already supplied canonically.
|
|
57
|
+
if (canonicalKey in out) continue;
|
|
58
|
+
out[canonicalKey] = value;
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function canonicalizeToolCall(toolName, toolInput) {
|
|
64
|
+
return {
|
|
65
|
+
toolName: canonicalizeToolName(toolName),
|
|
66
|
+
toolInput: canonicalizeToolInput(toolInput),
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = { canonicalizeToolCall, canonicalizeToolName, canonicalizeToolInput, TOOL_NAME_MAP, ARG_KEY_MAP };
|
package/scripts/hook-runtime.js
CHANGED
|
@@ -51,13 +51,20 @@ function resolveCliCommand(subcommand) {
|
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
function resolveCodexCliCommand(subcommand) {
|
|
54
|
+
// Codex hooks live in user-global config. Pinning them to a disposable source
|
|
55
|
+
// worktree breaks every hook as soon as normal post-merge cleanup removes it.
|
|
56
|
+
// The stable @latest launcher stays offline at configuration time and survives
|
|
57
|
+
// cleanup. Checkout-pinned hooks remain available as an explicit dev override.
|
|
58
|
+
if (isSourceCheckout(PKG_ROOT) && process.env.THUMBGATE_CODEX_USE_SOURCE_RUNTIME === '1') {
|
|
59
|
+
return `node ${shellQuote(path.join(PKG_ROOT, 'bin', 'cli.js'))} ${subcommand}`;
|
|
60
|
+
}
|
|
61
|
+
if (isSourceCheckout(PKG_ROOT)) {
|
|
62
|
+
return publishedCliShellCommand('latest', [subcommand]);
|
|
63
|
+
}
|
|
54
64
|
const version = packageVersion();
|
|
55
65
|
if (publishedHookCommandsAvailable(version)) {
|
|
56
66
|
return publishedCliShellCommand('latest', [subcommand]);
|
|
57
67
|
}
|
|
58
|
-
if (isSourceCheckout(PKG_ROOT)) {
|
|
59
|
-
return `node ${shellQuote(path.join(PKG_ROOT, 'bin', 'cli.js'))} ${subcommand}`;
|
|
60
|
-
}
|
|
61
68
|
return publishedCliShellCommand('latest', [subcommand]);
|
|
62
69
|
}
|
|
63
70
|
|
package/scripts/jsonl-watcher.js
CHANGED
|
@@ -71,6 +71,7 @@ function ingestEntry(entry) {
|
|
|
71
71
|
whatWorked: entry.whatWorked || undefined,
|
|
72
72
|
tags: [...(entry.tags || []), WATCHER_SOURCE_TAG, `bridged-from:${entry.source}`],
|
|
73
73
|
skill: entry.skill || undefined,
|
|
74
|
+
reviewOrigin: entry.reviewOrigin || 'imported',
|
|
74
75
|
});
|
|
75
76
|
|
|
76
77
|
return result;
|
package/scripts/lesson-db.js
CHANGED
|
@@ -15,14 +15,16 @@
|
|
|
15
15
|
const path = require('node:path');
|
|
16
16
|
const fs = require('node:fs');
|
|
17
17
|
const { readJsonl } = require('./fs-utils');
|
|
18
|
+
const { resolveFeedbackDir } = require('./feedback-paths');
|
|
18
19
|
|
|
19
|
-
|
|
20
|
-
|
|
20
|
+
function resolveDefaultDbPath(options = {}) {
|
|
21
|
+
return path.join(resolveFeedbackDir(options), 'lessons.sqlite');
|
|
22
|
+
}
|
|
21
23
|
|
|
22
24
|
/** @returns {import('better-sqlite3').Database} */
|
|
23
25
|
function initDB(dbPath) {
|
|
24
26
|
const Database = require('better-sqlite3');
|
|
25
|
-
const resolvedPath = dbPath || process.env.LESSON_DB_PATH ||
|
|
27
|
+
const resolvedPath = dbPath || process.env.LESSON_DB_PATH || resolveDefaultDbPath();
|
|
26
28
|
|
|
27
29
|
// Ensure parent directory exists
|
|
28
30
|
const dir = path.dirname(resolvedPath);
|
|
@@ -643,7 +645,7 @@ function safeParseTags(tagsStr) {
|
|
|
643
645
|
}
|
|
644
646
|
}
|
|
645
647
|
|
|
646
|
-
|
|
648
|
+
const lessonDbApi = {
|
|
647
649
|
initDB,
|
|
648
650
|
upsertLesson,
|
|
649
651
|
upsertSession,
|
|
@@ -655,5 +657,14 @@ module.exports = {
|
|
|
655
657
|
getStats,
|
|
656
658
|
getStatsFromDB,
|
|
657
659
|
backfillFromJsonl,
|
|
658
|
-
|
|
660
|
+
resolveDefaultDbPath,
|
|
659
661
|
};
|
|
662
|
+
|
|
663
|
+
// Preserve the public property while resolving it at access time so callers
|
|
664
|
+
// cannot cache a package-checkout path before project selection is known.
|
|
665
|
+
Object.defineProperty(lessonDbApi, 'DEFAULT_DB_PATH', {
|
|
666
|
+
enumerable: true,
|
|
667
|
+
get: resolveDefaultDbPath,
|
|
668
|
+
});
|
|
669
|
+
|
|
670
|
+
module.exports = lessonDbApi;
|