thumbgate 1.29.2 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +54 -16
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +143 -14
- package/adapters/opencode/opencode.json +1 -1
- package/bench/observability-eval-suite.json +2 -2
- package/bin/cli.js +154 -36
- package/config/evals/generation-quality-golden.json +95 -0
- package/config/evals/rag-answer-quality-golden.json +91 -0
- package/config/evals/retrieval-hybrid-ablation.json +66 -0
- package/config/evals/retrieval-ranking-golden.json +522 -0
- package/config/gates/default.json +217 -50
- package/config/mcp-allowlists.json +215 -185
- package/config/model-tiers.json +7 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/glama.json +6 -0
- package/package.json +94 -11
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +22 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/hero-thumbs.svg +68 -0
- package/public/assets/diagrams/loop.svg +40 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/self-improving-thumbs-loop.svg +105 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/compare.html +1 -0
- package/public/dashboard.html +126 -28
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +143 -13
- package/public/numbers.html +3 -2
- package/public/pricing.html +143 -30
- package/public/whitepaper.html +189 -0
- package/scripts/a-plus-evidence-scorecard.js +303 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/async-eval-observability.js +36 -11
- package/scripts/audit-trail.js +37 -1
- package/scripts/auto-promote-gates.js +149 -34
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/colbert-style-maxsim.js +236 -0
- package/scripts/cross-encoder-reranker.js +359 -126
- package/scripts/dashboard-chat.js +350 -17
- package/scripts/document-intake.js +283 -7
- package/scripts/eval-quality-suite.js +204 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +359 -189
- package/scripts/feedback-paths.js +32 -13
- package/scripts/feedback-quality.js +53 -0
- package/scripts/filesystem-search.js +17 -7
- package/scripts/gates-engine.js +98 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/harness-tool-names.js +70 -0
- package/scripts/hook-runtime.js +10 -3
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-db.js +16 -5
- package/scripts/lesson-embedding-index.js +67 -20
- package/scripts/lesson-embedding-maintenance.js +177 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-reranker.js +55 -9
- package/scripts/lesson-retrieval.js +375 -32
- package/scripts/lesson-search.js +48 -11
- package/scripts/llm-client.js +304 -15
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/model-tier-router.js +593 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/pragmatic-hybrid-search.js +379 -0
- package/scripts/published-cli.js +11 -1
- package/scripts/rag-document-pipeline.js +461 -0
- package/scripts/rag-structured-output.js +441 -0
- package/scripts/ragas-style-metrics.js +351 -0
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/request-envelope.js +178 -0
- package/scripts/rerank-pipeline.js +370 -0
- package/scripts/rerank-quality-eval.js +155 -0
- package/scripts/retrieval-hybrid-ablation.js +120 -0
- package/scripts/retrieval-quality-tier.js +118 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/secret-scanner.js +395 -4
- package/scripts/self-distill-agent.js +7 -1
- package/scripts/self-healing-check.js +25 -0
- package/scripts/skill-packs.js +183 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/statusline-links.js +1 -1
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/statusline.sh +8 -1
- package/scripts/telemetry-analytics.js +13 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/thumbgate-search.js +98 -6
- package/scripts/tier-budget-guard.js +186 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +95 -1
- package/scripts/vector-store.js +108 -4
- package/scripts/verify-marketing-pages-deployed.js +85 -3
- package/server.json +44 -0
- package/smithery.yaml +17 -0
- package/src/api/server.js +424 -99
|
@@ -22,6 +22,8 @@
|
|
|
22
22
|
* --json machine-readable report on stdout, suitable for piping
|
|
23
23
|
* into the GitHub Actions PR comment step.
|
|
24
24
|
* --quiet suppress per-route lines; only print the final summary.
|
|
25
|
+
* --max-attempts=N retry only transient transport/5xx failures.
|
|
26
|
+
* --retry-delay-ms=N bounded delay between transient retries.
|
|
25
27
|
*
|
|
26
28
|
* Exit code is 0 when every page passes, 1 if any sentinel is missing
|
|
27
29
|
* or any route returns non-200.
|
|
@@ -32,6 +34,8 @@ const path = require('node:path');
|
|
|
32
34
|
|
|
33
35
|
const DEFAULT_PROD_URL = 'https://thumbgate-production.up.railway.app';
|
|
34
36
|
const DEFAULT_TIMEOUT_MS = 12000;
|
|
37
|
+
const DEFAULT_MAX_ATTEMPTS = 2;
|
|
38
|
+
const DEFAULT_RETRY_DELAY_MS = 500;
|
|
35
39
|
const DEFAULT_MANIFEST_PATH = path.resolve(__dirname, '..', 'config', 'post-deploy-marketing-pages.json');
|
|
36
40
|
|
|
37
41
|
function parseArgs(argv = []) {
|
|
@@ -41,6 +45,8 @@ function parseArgs(argv = []) {
|
|
|
41
45
|
json: false,
|
|
42
46
|
quiet: false,
|
|
43
47
|
timeoutMs: DEFAULT_TIMEOUT_MS,
|
|
48
|
+
maxAttempts: DEFAULT_MAX_ATTEMPTS,
|
|
49
|
+
retryDelayMs: DEFAULT_RETRY_DELAY_MS,
|
|
44
50
|
};
|
|
45
51
|
for (const arg of argv) {
|
|
46
52
|
if (arg === '--json') out.json = true;
|
|
@@ -50,6 +56,12 @@ function parseArgs(argv = []) {
|
|
|
50
56
|
else if (arg.startsWith('--timeout-ms=')) {
|
|
51
57
|
const n = Number(arg.slice('--timeout-ms='.length));
|
|
52
58
|
if (Number.isFinite(n) && n > 0) out.timeoutMs = n;
|
|
59
|
+
} else if (arg.startsWith('--max-attempts=')) {
|
|
60
|
+
const n = Number(arg.slice('--max-attempts='.length));
|
|
61
|
+
if (Number.isInteger(n) && n > 0 && n <= 5) out.maxAttempts = n;
|
|
62
|
+
} else if (arg.startsWith('--retry-delay-ms=')) {
|
|
63
|
+
const n = Number(arg.slice('--retry-delay-ms='.length));
|
|
64
|
+
if (Number.isInteger(n) && n >= 0 && n <= 5000) out.retryDelayMs = n;
|
|
53
65
|
}
|
|
54
66
|
}
|
|
55
67
|
return out;
|
|
@@ -80,7 +92,7 @@ function loadManifest(manifestPath) {
|
|
|
80
92
|
return parsed;
|
|
81
93
|
}
|
|
82
94
|
|
|
83
|
-
async function
|
|
95
|
+
async function probePageOnce({ prodUrl, route, sentinel, mustNotContain = [], userAgent, fetchImpl, timeoutMs }) {
|
|
84
96
|
if (typeof fetchImpl !== 'function') {
|
|
85
97
|
return { route, ok: false, error: 'fetch_unavailable' };
|
|
86
98
|
}
|
|
@@ -97,7 +109,10 @@ async function probePage({ prodUrl, route, sentinel, mustNotContain = [], userAg
|
|
|
97
109
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
98
110
|
},
|
|
99
111
|
});
|
|
100
|
-
|
|
112
|
+
// Keep body-read transport failures visible to the outer retry classifier.
|
|
113
|
+
// Treating a disconnected 2xx stream as an empty body would incorrectly
|
|
114
|
+
// turn an operational failure into a deterministic sentinel mismatch.
|
|
115
|
+
const body = await res.text();
|
|
101
116
|
const sentinelPresent = body.includes(sentinel);
|
|
102
117
|
const forbiddenPresent = Array.isArray(mustNotContain)
|
|
103
118
|
? mustNotContain.filter((value) => body.includes(value))
|
|
@@ -122,7 +137,67 @@ async function probePage({ prodUrl, route, sentinel, mustNotContain = [], userAg
|
|
|
122
137
|
}
|
|
123
138
|
}
|
|
124
139
|
|
|
125
|
-
|
|
140
|
+
function isTransientProbeFailure(result = {}) {
|
|
141
|
+
if (result.ok) return false;
|
|
142
|
+
if (result.error && result.error !== 'fetch_unavailable') return true;
|
|
143
|
+
return result.status === 429 || result.status >= 500;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function waitForRetry(delayMs) {
|
|
147
|
+
if (!delayMs) return Promise.resolve();
|
|
148
|
+
return new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function probePage({
|
|
152
|
+
prodUrl,
|
|
153
|
+
route,
|
|
154
|
+
sentinel,
|
|
155
|
+
mustNotContain = [],
|
|
156
|
+
userAgent,
|
|
157
|
+
fetchImpl = globalThis.fetch,
|
|
158
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
159
|
+
maxAttempts = DEFAULT_MAX_ATTEMPTS,
|
|
160
|
+
retryDelayMs = DEFAULT_RETRY_DELAY_MS,
|
|
161
|
+
}) {
|
|
162
|
+
const boundedAttempts = Number.isInteger(maxAttempts)
|
|
163
|
+
? Math.min(Math.max(maxAttempts, 1), 5)
|
|
164
|
+
: DEFAULT_MAX_ATTEMPTS;
|
|
165
|
+
let result;
|
|
166
|
+
for (let attempt = 1; attempt <= boundedAttempts; attempt += 1) {
|
|
167
|
+
// eslint-disable-next-line no-await-in-loop
|
|
168
|
+
result = await probePageOnce({
|
|
169
|
+
prodUrl,
|
|
170
|
+
route,
|
|
171
|
+
sentinel,
|
|
172
|
+
mustNotContain,
|
|
173
|
+
userAgent,
|
|
174
|
+
fetchImpl,
|
|
175
|
+
timeoutMs,
|
|
176
|
+
});
|
|
177
|
+
if (!isTransientProbeFailure(result) || attempt === boundedAttempts) {
|
|
178
|
+
return {
|
|
179
|
+
...result,
|
|
180
|
+
attempts: attempt,
|
|
181
|
+
recoveredAfterRetry: Boolean(result.ok && attempt > 1),
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
// A deploy can pass /health while the first buyer-page request still
|
|
185
|
+
// wakes a cold instance. Retry only transient failures; content contract
|
|
186
|
+
// mismatches remain immediate, deterministic failures.
|
|
187
|
+
// eslint-disable-next-line no-await-in-loop
|
|
188
|
+
await waitForRetry(retryDelayMs);
|
|
189
|
+
}
|
|
190
|
+
return { ...result, attempts: boundedAttempts, recoveredAfterRetry: false };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function runVerification({
|
|
194
|
+
prodUrl,
|
|
195
|
+
manifestPath,
|
|
196
|
+
fetchImpl = globalThis.fetch,
|
|
197
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
198
|
+
maxAttempts = DEFAULT_MAX_ATTEMPTS,
|
|
199
|
+
retryDelayMs = DEFAULT_RETRY_DELAY_MS,
|
|
200
|
+
} = {}) {
|
|
126
201
|
const manifest = loadManifest(manifestPath);
|
|
127
202
|
const results = [];
|
|
128
203
|
// Sequential is fine — manifest has <20 entries; parallelism would
|
|
@@ -137,6 +212,8 @@ async function runVerification({ prodUrl, manifestPath, fetchImpl = globalThis.f
|
|
|
137
212
|
userAgent: entry.userAgent,
|
|
138
213
|
fetchImpl,
|
|
139
214
|
timeoutMs,
|
|
215
|
+
maxAttempts,
|
|
216
|
+
retryDelayMs,
|
|
140
217
|
});
|
|
141
218
|
results.push({ ...result, sentinel: entry.sentinel, description: entry.description });
|
|
142
219
|
}
|
|
@@ -183,6 +260,8 @@ async function main(argv) {
|
|
|
183
260
|
prodUrl: args.prodUrl,
|
|
184
261
|
manifestPath: args.manifestPath,
|
|
185
262
|
timeoutMs: args.timeoutMs,
|
|
263
|
+
maxAttempts: args.maxAttempts,
|
|
264
|
+
retryDelayMs: args.retryDelayMs,
|
|
186
265
|
});
|
|
187
266
|
} catch (error) {
|
|
188
267
|
process.stderr.write(`verify-marketing-pages-deployed FAILED: ${error.message}\n`);
|
|
@@ -199,9 +278,12 @@ async function main(argv) {
|
|
|
199
278
|
module.exports = {
|
|
200
279
|
DEFAULT_PROD_URL,
|
|
201
280
|
DEFAULT_TIMEOUT_MS,
|
|
281
|
+
DEFAULT_MAX_ATTEMPTS,
|
|
282
|
+
DEFAULT_RETRY_DELAY_MS,
|
|
202
283
|
DEFAULT_MANIFEST_PATH,
|
|
203
284
|
parseArgs,
|
|
204
285
|
loadManifest,
|
|
286
|
+
isTransientProbeFailure,
|
|
205
287
|
probePage,
|
|
206
288
|
runVerification,
|
|
207
289
|
renderHuman,
|
package/server.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
|
+
"name": "io.github.IgorGanapolsky/thumbgate",
|
|
4
|
+
"description": "ThumbGate pre-action gates for AI coding agents. Captures thumbs-up/down feedback as lessons, promotes repeated failures into prevention rules, and evaluates tool calls over stdio MCP before they run.",
|
|
5
|
+
"title": "ThumbGate",
|
|
6
|
+
"websiteUrl": "https://thumbgate.ai",
|
|
7
|
+
"repository": {
|
|
8
|
+
"source": "github",
|
|
9
|
+
"url": "https://github.com/IgorGanapolsky/ThumbGate"
|
|
10
|
+
},
|
|
11
|
+
"version": "1.31.0",
|
|
12
|
+
"packages": [
|
|
13
|
+
{
|
|
14
|
+
"registryType": "npm",
|
|
15
|
+
"registryBaseUrl": "https://registry.npmjs.org",
|
|
16
|
+
"identifier": "thumbgate",
|
|
17
|
+
"version": "1.31.0",
|
|
18
|
+
"runtimeHint": "npx",
|
|
19
|
+
"runtimeArguments": [
|
|
20
|
+
{
|
|
21
|
+
"type": "positional",
|
|
22
|
+
"value": "-y"
|
|
23
|
+
}
|
|
24
|
+
],
|
|
25
|
+
"packageArguments": [
|
|
26
|
+
{
|
|
27
|
+
"type": "positional",
|
|
28
|
+
"value": "serve"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"transport": {
|
|
32
|
+
"type": "stdio"
|
|
33
|
+
},
|
|
34
|
+
"environmentVariables": [
|
|
35
|
+
{
|
|
36
|
+
"name": "THUMBGATE_MCP_PROFILE",
|
|
37
|
+
"description": "Optional MCP tool profile override (essential | default | readonly | locked). Omit to keep the factory least-privilege essential profile; only set when expanded tools are intentional.",
|
|
38
|
+
"isRequired": false,
|
|
39
|
+
"format": "string"
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|
package/smithery.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
startCommand:
|
|
2
|
+
type: "stdio"
|
|
3
|
+
configSchema:
|
|
4
|
+
type: "object"
|
|
5
|
+
properties:
|
|
6
|
+
mcpProfile:
|
|
7
|
+
type: "string"
|
|
8
|
+
description: "MCP profile to use (default, readonly, locked)"
|
|
9
|
+
default: "default"
|
|
10
|
+
required: []
|
|
11
|
+
commandFunction:
|
|
12
|
+
command: "npx"
|
|
13
|
+
args:
|
|
14
|
+
- "-y"
|
|
15
|
+
- "thumbgate"
|
|
16
|
+
- "serve"
|
|
17
|
+
description: "ThumbGate — pre-action gates for AI coding agents. Turns thumbs-down feedback into prevention rules and evaluates tool calls over stdio MCP (npx -y thumbgate serve) before they run. Not the HTTP API (npm start). Works with Claude Code, Cursor, Codex, Gemini CLI, Amp, OpenCode, and any MCP-compatible agent."
|