thumbgate 1.29.2 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +54 -16
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +143 -14
- package/adapters/opencode/opencode.json +1 -1
- package/bench/observability-eval-suite.json +2 -2
- package/bin/cli.js +154 -36
- package/config/evals/generation-quality-golden.json +95 -0
- package/config/evals/rag-answer-quality-golden.json +91 -0
- package/config/evals/retrieval-hybrid-ablation.json +66 -0
- package/config/evals/retrieval-ranking-golden.json +522 -0
- package/config/gates/default.json +217 -50
- package/config/mcp-allowlists.json +215 -185
- package/config/model-tiers.json +7 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/glama.json +6 -0
- package/package.json +94 -11
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +22 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/hero-thumbs.svg +68 -0
- package/public/assets/diagrams/loop.svg +40 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/self-improving-thumbs-loop.svg +105 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/compare.html +1 -0
- package/public/dashboard.html +126 -28
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +143 -13
- package/public/numbers.html +3 -2
- package/public/pricing.html +143 -30
- package/public/whitepaper.html +189 -0
- package/scripts/a-plus-evidence-scorecard.js +303 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/async-eval-observability.js +36 -11
- package/scripts/audit-trail.js +37 -1
- package/scripts/auto-promote-gates.js +149 -34
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/colbert-style-maxsim.js +236 -0
- package/scripts/cross-encoder-reranker.js +359 -126
- package/scripts/dashboard-chat.js +350 -17
- package/scripts/document-intake.js +283 -7
- package/scripts/eval-quality-suite.js +204 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +359 -189
- package/scripts/feedback-paths.js +32 -13
- package/scripts/feedback-quality.js +53 -0
- package/scripts/filesystem-search.js +17 -7
- package/scripts/gates-engine.js +98 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/harness-tool-names.js +70 -0
- package/scripts/hook-runtime.js +10 -3
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-db.js +16 -5
- package/scripts/lesson-embedding-index.js +67 -20
- package/scripts/lesson-embedding-maintenance.js +177 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-reranker.js +55 -9
- package/scripts/lesson-retrieval.js +375 -32
- package/scripts/lesson-search.js +48 -11
- package/scripts/llm-client.js +304 -15
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/model-tier-router.js +593 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/pragmatic-hybrid-search.js +379 -0
- package/scripts/published-cli.js +11 -1
- package/scripts/rag-document-pipeline.js +461 -0
- package/scripts/rag-structured-output.js +441 -0
- package/scripts/ragas-style-metrics.js +351 -0
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/request-envelope.js +178 -0
- package/scripts/rerank-pipeline.js +370 -0
- package/scripts/rerank-quality-eval.js +155 -0
- package/scripts/retrieval-hybrid-ablation.js +120 -0
- package/scripts/retrieval-quality-tier.js +118 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/secret-scanner.js +395 -4
- package/scripts/self-distill-agent.js +7 -1
- package/scripts/self-healing-check.js +25 -0
- package/scripts/skill-packs.js +183 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/statusline-links.js +1 -1
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/statusline.sh +8 -1
- package/scripts/telemetry-analytics.js +13 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/thumbgate-search.js +98 -6
- package/scripts/tier-budget-guard.js +186 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +95 -1
- package/scripts/vector-store.js +108 -4
- package/scripts/verify-marketing-pages-deployed.js +85 -3
- package/server.json +44 -0
- package/smithery.yaml +17 -0
- package/src/api/server.js +424 -99
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Hard cost + latency budgets for model tier routing.
|
|
5
|
+
*
|
|
6
|
+
* Complements FrontierBudget (session token cap) with:
|
|
7
|
+
* - per-request max cost (cents)
|
|
8
|
+
* - per-request max latency budget (ms) for planning/degrade
|
|
9
|
+
* - max frontier invocations per day (process-local counter)
|
|
10
|
+
*
|
|
11
|
+
* Env (optional):
|
|
12
|
+
* THUMBGATE_MAX_COST_CENTS_PER_REQUEST
|
|
13
|
+
* THUMBGATE_MAX_LATENCY_MS
|
|
14
|
+
* THUMBGATE_MAX_FRONTIER_PER_DAY
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const { TIERS, classifyTask, FrontierBudget } = require('./model-tier-router');
|
|
18
|
+
|
|
19
|
+
const DEFAULTS = Object.freeze({
|
|
20
|
+
maxCostCentsPerRequest: 25, // $0.25
|
|
21
|
+
maxLatencyMs: 30_000,
|
|
22
|
+
maxFrontierPerDay: 50,
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
/** @type {Map<string, number>} dayKey → frontier invocation count */
|
|
26
|
+
const frontierDayCounts = new Map();
|
|
27
|
+
|
|
28
|
+
function dayKey(now = Date.now()) {
|
|
29
|
+
return new Date(now).toISOString().slice(0, 10);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function readEnvNumber(name, fallback) {
|
|
33
|
+
const raw = process.env[name];
|
|
34
|
+
if (raw == null || raw === '') return fallback;
|
|
35
|
+
const n = Number(raw);
|
|
36
|
+
return Number.isFinite(n) ? n : fallback;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function getBudgetConfig(overrides = {}) {
|
|
40
|
+
return {
|
|
41
|
+
maxCostCentsPerRequest: overrides.maxCostCentsPerRequest
|
|
42
|
+
?? readEnvNumber('THUMBGATE_MAX_COST_CENTS_PER_REQUEST', DEFAULTS.maxCostCentsPerRequest),
|
|
43
|
+
maxLatencyMs: overrides.maxLatencyMs
|
|
44
|
+
?? readEnvNumber('THUMBGATE_MAX_LATENCY_MS', DEFAULTS.maxLatencyMs),
|
|
45
|
+
maxFrontierPerDay: overrides.maxFrontierPerDay
|
|
46
|
+
?? readEnvNumber('THUMBGATE_MAX_FRONTIER_PER_DAY', DEFAULTS.maxFrontierPerDay),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Estimate request cost cents from tier + token budget.
|
|
52
|
+
* @param {string} tier
|
|
53
|
+
* @param {number} estimatedTokens
|
|
54
|
+
*/
|
|
55
|
+
function estimateTierCostCents(tier, estimatedTokens = 4000) {
|
|
56
|
+
const t = TIERS[tier] || TIERS.mini;
|
|
57
|
+
// Base: ~$3/M input + $15/M out blended as ~$6/M total for coding
|
|
58
|
+
const basePerM = 6;
|
|
59
|
+
const usd = (estimatedTokens / 1e6) * basePerM * (t.costMultiplier ?? 1);
|
|
60
|
+
return Number((usd * 100).toFixed(4));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Enforce hard budgets on a classification result.
|
|
65
|
+
*
|
|
66
|
+
* @param {object} task — same shape as classifyTask
|
|
67
|
+
* @param {object} [options]
|
|
68
|
+
* @param {object} [options.classification] — precomputed classifyTask result
|
|
69
|
+
* @param {FrontierBudget} [options.frontierBudget]
|
|
70
|
+
* @param {number} [options.estimatedTokens]
|
|
71
|
+
* @param {number} [options.nowMs]
|
|
72
|
+
* @returns {{
|
|
73
|
+
* allowed: boolean,
|
|
74
|
+
* tier: string,
|
|
75
|
+
* action: 'allow'|'degrade'|'deny',
|
|
76
|
+
* reasons: string[],
|
|
77
|
+
* classification: object,
|
|
78
|
+
* estimatedCostCents: number,
|
|
79
|
+
* budget: object,
|
|
80
|
+
* }}
|
|
81
|
+
*/
|
|
82
|
+
function enforceTierBudgets(task = {}, options = {}) {
|
|
83
|
+
const config = getBudgetConfig(options);
|
|
84
|
+
const classification = options.classification || classifyTask(task);
|
|
85
|
+
let tier = classification.tier;
|
|
86
|
+
const reasons = [];
|
|
87
|
+
const estimatedTokens = Number(options.estimatedTokens) || 4000;
|
|
88
|
+
let estimatedCostCents = estimateTierCostCents(tier, estimatedTokens);
|
|
89
|
+
let action = 'allow';
|
|
90
|
+
|
|
91
|
+
// Daily frontier cap (process-local)
|
|
92
|
+
if (tier === 'frontier') {
|
|
93
|
+
const key = dayKey(options.nowMs);
|
|
94
|
+
const used = frontierDayCounts.get(key) || 0;
|
|
95
|
+
if (used >= config.maxFrontierPerDay) {
|
|
96
|
+
reasons.push(`frontier_daily_cap:${used}/${config.maxFrontierPerDay}`);
|
|
97
|
+
tier = 'mini';
|
|
98
|
+
action = 'degrade';
|
|
99
|
+
estimatedCostCents = estimateTierCostCents(tier, estimatedTokens);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Per-request cost cap → degrade tier then deny if still over
|
|
104
|
+
if (estimatedCostCents > config.maxCostCentsPerRequest) {
|
|
105
|
+
if (tier === 'frontier') {
|
|
106
|
+
reasons.push(`cost_over_cap_degrade:${estimatedCostCents}>${config.maxCostCentsPerRequest}`);
|
|
107
|
+
tier = 'mini';
|
|
108
|
+
action = 'degrade';
|
|
109
|
+
estimatedCostCents = estimateTierCostCents(tier, estimatedTokens);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (estimatedCostCents > config.maxCostCentsPerRequest && tier !== 'nano' && tier !== 'localFrontier') {
|
|
113
|
+
reasons.push(`cost_over_cap_degrade_nano:${estimatedCostCents}>${config.maxCostCentsPerRequest}`);
|
|
114
|
+
tier = 'nano';
|
|
115
|
+
action = 'degrade';
|
|
116
|
+
estimatedCostCents = estimateTierCostCents(tier, estimatedTokens);
|
|
117
|
+
}
|
|
118
|
+
if (estimatedCostCents > config.maxCostCentsPerRequest && (TIERS[tier]?.costMultiplier || 0) > 0) {
|
|
119
|
+
reasons.push(`cost_deny:${estimatedCostCents}>${config.maxCostCentsPerRequest}`);
|
|
120
|
+
action = 'deny';
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Session frontier token budget
|
|
124
|
+
const frontierBudget = options.frontierBudget || null;
|
|
125
|
+
if (frontierBudget && tier === 'frontier' && typeof frontierBudget.canSpend === 'function') {
|
|
126
|
+
const check = frontierBudget.canSpend(estimatedTokens, task.reason || classification.reason || 'routed_frontier');
|
|
127
|
+
if (!check.allowed) {
|
|
128
|
+
reasons.push(`frontier_session_budget:${check.reason}`);
|
|
129
|
+
tier = 'mini';
|
|
130
|
+
action = action === 'deny' ? 'deny' : 'degrade';
|
|
131
|
+
estimatedCostCents = estimateTierCostCents(tier, estimatedTokens);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Latency budget is advisory for planning (caller may still enforce timeouts)
|
|
136
|
+
if (Number(task.expectedLatencyMs) > config.maxLatencyMs) {
|
|
137
|
+
reasons.push(`latency_budget_exceeded_plan:${task.expectedLatencyMs}>${config.maxLatencyMs}`);
|
|
138
|
+
if (tier === 'frontier') {
|
|
139
|
+
tier = 'mini';
|
|
140
|
+
action = action === 'deny' ? 'deny' : 'degrade';
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const allowed = action !== 'deny';
|
|
145
|
+
return {
|
|
146
|
+
allowed,
|
|
147
|
+
tier,
|
|
148
|
+
action,
|
|
149
|
+
reasons,
|
|
150
|
+
classification,
|
|
151
|
+
estimatedCostCents,
|
|
152
|
+
budget: {
|
|
153
|
+
...config,
|
|
154
|
+
estimatedTokens,
|
|
155
|
+
},
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Record a frontier invocation against the daily counter (call only when actually used).
|
|
161
|
+
*/
|
|
162
|
+
function recordFrontierInvocation(nowMs = Date.now()) {
|
|
163
|
+
const key = dayKey(nowMs);
|
|
164
|
+
frontierDayCounts.set(key, (frontierDayCounts.get(key) || 0) + 1);
|
|
165
|
+
return frontierDayCounts.get(key);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Test helper */
|
|
169
|
+
function _resetFrontierDayCounts() {
|
|
170
|
+
frontierDayCounts.clear();
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function getFrontierDayUsage(nowMs = Date.now()) {
|
|
174
|
+
const key = dayKey(nowMs);
|
|
175
|
+
return { day: key, count: frontierDayCounts.get(key) || 0 };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
module.exports = {
|
|
179
|
+
DEFAULTS,
|
|
180
|
+
getBudgetConfig,
|
|
181
|
+
estimateTierCostCents,
|
|
182
|
+
enforceTierBudgets,
|
|
183
|
+
recordFrontierInvocation,
|
|
184
|
+
getFrontierDayUsage,
|
|
185
|
+
_resetFrontierDayCounts,
|
|
186
|
+
};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const crypto = require('node:crypto');
|
|
5
|
+
const fs = require('node:fs');
|
|
6
|
+
const path = require('node:path');
|
|
7
|
+
const { resolveFeedbackDir } = require('./feedback-paths');
|
|
8
|
+
const { readJsonl } = require('./fs-utils');
|
|
9
|
+
|
|
10
|
+
function getKpiLogPath(options = {}) {
|
|
11
|
+
return path.join(
|
|
12
|
+
options.feedbackDir ? path.resolve(options.feedbackDir) : resolveFeedbackDir(),
|
|
13
|
+
'tool-kpi.jsonl',
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function recordToolCall({
|
|
18
|
+
toolName,
|
|
19
|
+
serverName,
|
|
20
|
+
latencyMs,
|
|
21
|
+
success,
|
|
22
|
+
agentId,
|
|
23
|
+
metadata,
|
|
24
|
+
feedbackDir,
|
|
25
|
+
} = {}) {
|
|
26
|
+
const logPath = getKpiLogPath({ feedbackDir });
|
|
27
|
+
fs.mkdirSync(path.dirname(logPath), { recursive: true });
|
|
28
|
+
const entry = {
|
|
29
|
+
id: `kpi_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`,
|
|
30
|
+
timestamp: new Date().toISOString(),
|
|
31
|
+
toolName: toolName || 'unknown',
|
|
32
|
+
serverName: serverName || 'default',
|
|
33
|
+
latencyMs: typeof latencyMs === 'number' ? latencyMs : 0,
|
|
34
|
+
success: success !== false,
|
|
35
|
+
agentId: agentId || 'unknown',
|
|
36
|
+
metadata: metadata || {},
|
|
37
|
+
};
|
|
38
|
+
fs.appendFileSync(logPath, `${JSON.stringify(entry)}\n`);
|
|
39
|
+
return entry;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function percentile(sorted, quantile) {
|
|
43
|
+
if (sorted.length === 0) return 0;
|
|
44
|
+
const index = Math.ceil((quantile / 100) * sorted.length) - 1;
|
|
45
|
+
return sorted[Math.max(0, index)];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function computeToolKpis({ periodHours = 24, feedbackDir } = {}) {
|
|
49
|
+
const entries = readJsonl(getKpiLogPath({ feedbackDir }));
|
|
50
|
+
const cutoff = Date.now() - periodHours * 60 * 60 * 1000;
|
|
51
|
+
const recent = entries.filter((entry) => new Date(entry.timestamp).getTime() > cutoff);
|
|
52
|
+
const byTool = {};
|
|
53
|
+
for (const entry of recent) {
|
|
54
|
+
const key = entry.toolName;
|
|
55
|
+
if (!byTool[key]) {
|
|
56
|
+
byTool[key] = {
|
|
57
|
+
toolName: key,
|
|
58
|
+
calls: [],
|
|
59
|
+
successes: 0,
|
|
60
|
+
failures: 0,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
byTool[key].calls.push(entry.latencyMs);
|
|
64
|
+
if (entry.success) byTool[key].successes += 1;
|
|
65
|
+
else byTool[key].failures += 1;
|
|
66
|
+
}
|
|
67
|
+
const tools = Object.values(byTool)
|
|
68
|
+
.map((tool) => {
|
|
69
|
+
const sorted = tool.calls.slice().sort((left, right) => left - right);
|
|
70
|
+
const total = tool.successes + tool.failures;
|
|
71
|
+
return {
|
|
72
|
+
toolName: tool.toolName,
|
|
73
|
+
requestCount: total,
|
|
74
|
+
successRate: total > 0 ? Math.round((tool.successes / total) * 1000) / 10 : 100,
|
|
75
|
+
p50: Math.round(percentile(sorted, 50)),
|
|
76
|
+
p90: Math.round(percentile(sorted, 90)),
|
|
77
|
+
p95: Math.round(percentile(sorted, 95)),
|
|
78
|
+
successes: tool.successes,
|
|
79
|
+
failures: tool.failures,
|
|
80
|
+
};
|
|
81
|
+
})
|
|
82
|
+
.sort((left, right) => right.requestCount - left.requestCount);
|
|
83
|
+
|
|
84
|
+
const byServer = {};
|
|
85
|
+
for (const entry of recent) {
|
|
86
|
+
const key = entry.serverName;
|
|
87
|
+
if (!byServer[key]) byServer[key] = { serverName: key, total: 0, successes: 0 };
|
|
88
|
+
byServer[key].total += 1;
|
|
89
|
+
if (entry.success) byServer[key].successes += 1;
|
|
90
|
+
}
|
|
91
|
+
const servers = Object.values(byServer).map((server) => ({
|
|
92
|
+
serverName: server.serverName,
|
|
93
|
+
totalCalls: server.total,
|
|
94
|
+
successRate: server.total > 0
|
|
95
|
+
? Math.round((server.successes / server.total) * 1000) / 10
|
|
96
|
+
: 100,
|
|
97
|
+
}));
|
|
98
|
+
return {
|
|
99
|
+
periodHours,
|
|
100
|
+
totalCalls: recent.length,
|
|
101
|
+
evidenceStatus: recent.length > 0 ? 'measured' : 'insufficient_evidence',
|
|
102
|
+
tools,
|
|
103
|
+
servers,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function getAtRiskTools({
|
|
108
|
+
successRateThreshold = 90,
|
|
109
|
+
p95Threshold = 500,
|
|
110
|
+
periodHours = 24,
|
|
111
|
+
feedbackDir,
|
|
112
|
+
} = {}) {
|
|
113
|
+
const { tools } = computeToolKpis({ periodHours, feedbackDir });
|
|
114
|
+
return tools.filter((tool) => tool.requestCount >= 3
|
|
115
|
+
&& (tool.successRate < successRateThreshold || tool.p95 > p95Threshold));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
module.exports = {
|
|
119
|
+
computeToolKpis,
|
|
120
|
+
getAtRiskTools,
|
|
121
|
+
getKpiLogPath,
|
|
122
|
+
percentile,
|
|
123
|
+
recordToolCall,
|
|
124
|
+
};
|
package/scripts/tool-registry.js
CHANGED
|
@@ -133,8 +133,20 @@ const TASK_OUTCOME_INPUT_SCHEMA = {
|
|
|
133
133
|
},
|
|
134
134
|
};
|
|
135
135
|
|
|
136
|
+
const MEMORY_SCOPE_SCHEMA = {
|
|
137
|
+
type: 'object',
|
|
138
|
+
additionalProperties: false,
|
|
139
|
+
required: ['entityId', 'projectId', 'processId', 'sessionId'],
|
|
140
|
+
properties: {
|
|
141
|
+
entityId: { type: 'string', minLength: 1 },
|
|
142
|
+
projectId: { type: 'string', minLength: 1 },
|
|
143
|
+
processId: { type: 'string', minLength: 1 },
|
|
144
|
+
sessionId: { type: 'string', minLength: 1 },
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
|
|
136
148
|
const TOOLS = [
|
|
137
|
-
|
|
149
|
+
destructiveTool({
|
|
138
150
|
name: 'capture_feedback',
|
|
139
151
|
description: 'Capture an up/down signal plus one line of why. Vague feedback is logged, then returned with a clarification prompt instead of memory promotion.',
|
|
140
152
|
inputSchema: {
|
|
@@ -218,6 +230,9 @@ const TOOLS = [
|
|
|
218
230
|
limit: { type: 'number', description: 'Maximum results to return (default 10)' },
|
|
219
231
|
category: { type: 'string', enum: ['error', 'learning', 'preference'] },
|
|
220
232
|
tags: { type: 'array', items: { type: 'string' }, description: 'Require all tags to be present on a lesson' },
|
|
233
|
+
scope: MEMORY_SCOPE_SCHEMA,
|
|
234
|
+
requireScope: { type: 'boolean', description: 'Fail closed unless a complete four-field scope is supplied.' },
|
|
235
|
+
includeShared: { type: 'boolean', description: 'Include explicitly shared memories with scoped results. Defaults true.' },
|
|
221
236
|
},
|
|
222
237
|
},
|
|
223
238
|
}),
|
|
@@ -230,6 +245,23 @@ const TOOLS = [
|
|
|
230
245
|
toolName: { type: 'string', description: 'The tool being called (e.g., Bash, Edit, Read)' },
|
|
231
246
|
actionContext: { type: 'string', description: 'Description of what the tool call is doing' },
|
|
232
247
|
maxResults: { type: 'number', description: 'Max lessons to return (default 5)' },
|
|
248
|
+
scope: MEMORY_SCOPE_SCHEMA,
|
|
249
|
+
requireScope: { type: 'boolean', description: 'Fail closed unless a complete four-field scope is supplied.' },
|
|
250
|
+
includeShared: { type: 'boolean', description: 'Include explicitly shared memories with scoped results. Defaults true.' },
|
|
251
|
+
filters: {
|
|
252
|
+
type: 'object',
|
|
253
|
+
description: 'Pre-filter the candidate corpus before lexical or vector ranking.',
|
|
254
|
+
properties: {
|
|
255
|
+
domain: { type: ['string', 'array'], items: { type: 'string' } },
|
|
256
|
+
signal: { type: ['string', 'array'], items: { type: 'string' } },
|
|
257
|
+
source: { type: ['string', 'array'], items: { type: 'string' } },
|
|
258
|
+
toolsUsed: { type: ['string', 'array'], items: { type: 'string' } },
|
|
259
|
+
tags: { type: 'array', items: { type: 'string' } },
|
|
260
|
+
requireAllTags: { type: 'boolean' },
|
|
261
|
+
},
|
|
262
|
+
},
|
|
263
|
+
queryRewrite: { type: 'boolean', description: 'Enable bounded deterministic synonym expansion when the original lexical match is weak. Defaults true.' },
|
|
264
|
+
includeRetrievalMeta: { type: 'boolean', description: 'Include path provenance such as lexical/dense pools and query variants.' },
|
|
233
265
|
},
|
|
234
266
|
required: ['toolName'],
|
|
235
267
|
},
|
|
@@ -258,6 +290,17 @@ const TOOLS = [
|
|
|
258
290
|
limit: { type: 'number', description: 'Maximum results to return (default 10)' },
|
|
259
291
|
source: { type: 'string', enum: ['all', 'feedback', 'context', 'rules', 'documents'], description: 'Restrict search to a single ThumbGate source.' },
|
|
260
292
|
signal: { type: 'string', enum: ['up', 'down', 'positive', 'negative'], description: 'Optional feedback-signal filter when searching feedback data.' },
|
|
293
|
+
filters: {
|
|
294
|
+
type: 'object',
|
|
295
|
+
description: 'Pre-filter imported documents before chunk ranking.',
|
|
296
|
+
properties: {
|
|
297
|
+
tags: { type: 'array', items: { type: 'string' } },
|
|
298
|
+
requireAllTags: { type: 'boolean' },
|
|
299
|
+
sourceFormat: { type: 'string' },
|
|
300
|
+
sourceType: { type: 'string' },
|
|
301
|
+
},
|
|
302
|
+
},
|
|
303
|
+
queryRewrite: { type: 'boolean', description: 'Enable bounded deterministic synonym expansion. Defaults true.' },
|
|
261
304
|
},
|
|
262
305
|
},
|
|
263
306
|
}),
|
|
@@ -904,6 +947,10 @@ const TOOLS = [
|
|
|
904
947
|
items: { type: 'string' },
|
|
905
948
|
description: 'Optional protected-file globs that require explicit approval before editing or publishing',
|
|
906
949
|
},
|
|
950
|
+
ttlMs: {
|
|
951
|
+
type: 'number',
|
|
952
|
+
description: 'Optional lease length in milliseconds. With it the scope becomes time-bounded authority (e.g. 90000 for "write under ./src for 90 seconds") and FAILS CLOSED on expiry: a lapsed lease authorises nothing until renewed. Omit for a permanent scope. Clamped to 60s..24h.',
|
|
953
|
+
},
|
|
907
954
|
workflowContract: {
|
|
908
955
|
type: 'object',
|
|
909
956
|
description: 'Optional deterministic workflow run contract. Supports workflowId, allowedBranches, blockedActions, requiredEvidence, and completionGate.',
|
|
@@ -1047,6 +1094,16 @@ const TOOLS = [
|
|
|
1047
1094
|
title: 'Record Verified Task Outcome',
|
|
1048
1095
|
description: 'Record an idempotent task-level outcome with verification evidence, tool correctness, policy behavior, latency, cost, and business KPI movement. A completed response without evidence is recorded as not working.',
|
|
1049
1096
|
inputSchema: TASK_OUTCOME_INPUT_SCHEMA,
|
|
1097
|
+
outputSchema: {
|
|
1098
|
+
type: 'object',
|
|
1099
|
+
additionalProperties: false,
|
|
1100
|
+
required: ['recorded', 'duplicate', 'receipt'],
|
|
1101
|
+
properties: {
|
|
1102
|
+
recorded: { type: 'boolean' },
|
|
1103
|
+
duplicate: { type: 'boolean' },
|
|
1104
|
+
receipt: { type: 'object', additionalProperties: true },
|
|
1105
|
+
},
|
|
1106
|
+
},
|
|
1050
1107
|
}),
|
|
1051
1108
|
readOnlyTool({
|
|
1052
1109
|
name: 'get_task_outcomes',
|
|
@@ -1070,6 +1127,22 @@ const TOOLS = [
|
|
|
1070
1127
|
additionalProperties: false,
|
|
1071
1128
|
properties: {},
|
|
1072
1129
|
},
|
|
1130
|
+
outputSchema: {
|
|
1131
|
+
type: 'object',
|
|
1132
|
+
additionalProperties: true,
|
|
1133
|
+
required: ['generatedAt', 'sampleSize', 'evidenceStatus', 'task', 'tools', 'safety', 'escalation', 'efficiency', 'businessOutcomes'],
|
|
1134
|
+
properties: {
|
|
1135
|
+
generatedAt: { type: 'string', format: 'date-time' },
|
|
1136
|
+
sampleSize: { type: 'integer', minimum: 0 },
|
|
1137
|
+
evidenceStatus: { type: 'string', enum: ['measured', 'insufficient_evidence'] },
|
|
1138
|
+
task: { type: 'object', additionalProperties: true },
|
|
1139
|
+
tools: { type: 'object', additionalProperties: true },
|
|
1140
|
+
safety: { type: 'object', additionalProperties: true },
|
|
1141
|
+
escalation: { type: 'object', additionalProperties: true },
|
|
1142
|
+
efficiency: { type: 'object', additionalProperties: true },
|
|
1143
|
+
businessOutcomes: { type: 'array' },
|
|
1144
|
+
},
|
|
1145
|
+
},
|
|
1073
1146
|
}),
|
|
1074
1147
|
destructiveTool({
|
|
1075
1148
|
name: 'request_human_escalation',
|
|
@@ -1237,6 +1310,27 @@ const TOOLS = [
|
|
|
1237
1310
|
},
|
|
1238
1311
|
},
|
|
1239
1312
|
}),
|
|
1313
|
+
readOnlyTool({
|
|
1314
|
+
name: 'gate_check',
|
|
1315
|
+
// Harnesses with a native pre-tool hook (Claude Code, Codex, Gemini, Forge) get hard
|
|
1316
|
+
// enforcement and never need this. MCP-only harnesses (Cline, Cursor, OpenCode) have no
|
|
1317
|
+
// interception point, so this exposes the SAME gates engine as a tool the agent calls
|
|
1318
|
+
// before acting. adapters/cline/.clinerules has instructed agents to call
|
|
1319
|
+
// `thumbgate.gate_check` since the adapter shipped — but the tool did not exist, so
|
|
1320
|
+
// that enforcement was inert. This makes the documented contract real.
|
|
1321
|
+
description: 'Evaluate a proposed tool call against ThumbGate policy BEFORE executing it. Returns decision "block" (do not run the action; surface the reason) or "allow". Use this when about to run a shell command, write/edit a file, or take any irreversible action.',
|
|
1322
|
+
inputSchema: {
|
|
1323
|
+
type: 'object',
|
|
1324
|
+
required: ['tool_name'],
|
|
1325
|
+
properties: {
|
|
1326
|
+
tool_name: { type: 'string', description: 'The tool about to be invoked (e.g. Bash, Write, Edit)' },
|
|
1327
|
+
tool_input: {
|
|
1328
|
+
type: 'object',
|
|
1329
|
+
description: 'The proposed arguments, e.g. { "command": "rm -rf /" } for Bash or { "file_path": "..." } for Write',
|
|
1330
|
+
},
|
|
1331
|
+
},
|
|
1332
|
+
},
|
|
1333
|
+
}),
|
|
1240
1334
|
readOnlyTool({
|
|
1241
1335
|
name: 'gate_stats',
|
|
1242
1336
|
description: 'Get gate enforcement statistics -- blocked count, warned count, top gates',
|
package/scripts/vector-store.js
CHANGED
|
@@ -80,6 +80,42 @@ function hasLocalTransformerProvider() {
|
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
+
function getOllamaEmbeddingConfig() {
|
|
84
|
+
const model = String(process.env.THUMBGATE_OLLAMA_EMBED_MODEL || '').trim();
|
|
85
|
+
let endpoint = String(
|
|
86
|
+
process.env.THUMBGATE_OLLAMA_ENDPOINT
|
|
87
|
+
|| process.env.OLLAMA_HOST
|
|
88
|
+
|| 'http://127.0.0.1:11434',
|
|
89
|
+
).trim();
|
|
90
|
+
if (endpoint && !/^https?:\/\//i.test(endpoint)) endpoint = `http://${endpoint}`;
|
|
91
|
+
while (endpoint.endsWith('/')) endpoint = endpoint.slice(0, -1);
|
|
92
|
+
return {
|
|
93
|
+
enabled: Boolean(model),
|
|
94
|
+
model,
|
|
95
|
+
endpoint,
|
|
96
|
+
timeoutMs: Math.max(
|
|
97
|
+
250,
|
|
98
|
+
Math.min(30_000, Number(process.env.THUMBGATE_OLLAMA_TIMEOUT_MS) || 10_000),
|
|
99
|
+
),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Synchronous capability check used by hot-path routing. This reports only
|
|
105
|
+
* semantic providers, never the deterministic feature-hash degradation.
|
|
106
|
+
*/
|
|
107
|
+
function hasSemanticEmbeddingProvider() {
|
|
108
|
+
if (process.env.THUMBGATE_VECTOR_STUB_EMBED === 'true') return true;
|
|
109
|
+
if (getOllamaEmbeddingConfig().enabled) return true;
|
|
110
|
+
if (hasLocalTransformerProvider()) return true;
|
|
111
|
+
try {
|
|
112
|
+
const config = resolveGeminiEmbeddingConfig();
|
|
113
|
+
return config.provider === 'coreai' || Boolean(config.enabled && config.apiKey);
|
|
114
|
+
} catch {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
83
119
|
function fnv1a32(value) {
|
|
84
120
|
let hash = 0x811c9dc5;
|
|
85
121
|
const bytes = Buffer.from(String(value), 'utf8');
|
|
@@ -98,7 +134,7 @@ function addHashedFeature(vector, feature, weight) {
|
|
|
98
134
|
}
|
|
99
135
|
|
|
100
136
|
function embedWithFeatureHash(text) {
|
|
101
|
-
const vector = Array(FEATURE_HASH_DIMENSIONS).fill(0);
|
|
137
|
+
const vector = new Array(FEATURE_HASH_DIMENSIONS).fill(0);
|
|
102
138
|
const tokens = String(text || '').toLowerCase().match(/[\p{L}\p{N}_-]+/gu) || [];
|
|
103
139
|
|
|
104
140
|
for (let index = 0; index < tokens.length; index += 1) {
|
|
@@ -249,6 +285,43 @@ async function embedWithCoreAI(text, options = {}) {
|
|
|
249
285
|
throw new Error('Core AI local service did not return a valid embedding');
|
|
250
286
|
}
|
|
251
287
|
|
|
288
|
+
async function embedWithOllama(text, options = {}) {
|
|
289
|
+
const config = getOllamaEmbeddingConfig();
|
|
290
|
+
if (!config.enabled) {
|
|
291
|
+
throw new Error('Ollama embeddings require THUMBGATE_OLLAMA_EMBED_MODEL');
|
|
292
|
+
}
|
|
293
|
+
if (typeof fetch !== 'function') {
|
|
294
|
+
throw new Error('Ollama embeddings require global fetch. Use Node 18.18+.');
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
let response;
|
|
298
|
+
try {
|
|
299
|
+
response = await fetch(`${config.endpoint}/api/embed`, {
|
|
300
|
+
method: 'POST',
|
|
301
|
+
headers: { 'Content-Type': 'application/json' },
|
|
302
|
+
body: JSON.stringify({
|
|
303
|
+
model: config.model,
|
|
304
|
+
input: String(text || ''),
|
|
305
|
+
truncate: true,
|
|
306
|
+
dimensions: options.outputDimensionality || undefined,
|
|
307
|
+
}),
|
|
308
|
+
signal: AbortSignal.timeout(config.timeoutMs),
|
|
309
|
+
});
|
|
310
|
+
} catch (error) {
|
|
311
|
+
throw new Error(`Ollama embedding service unavailable: ${error.message}`);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if (!response.ok) {
|
|
315
|
+
throw new Error(`Ollama embedding request failed: ${response.status} ${response.statusText}`);
|
|
316
|
+
}
|
|
317
|
+
const payload = await response.json();
|
|
318
|
+
const vector = Array.isArray(payload.embeddings) ? payload.embeddings[0] : null;
|
|
319
|
+
if (!Array.isArray(vector) || vector.length === 0) {
|
|
320
|
+
throw new Error('Ollama embedding response did not include vector values');
|
|
321
|
+
}
|
|
322
|
+
return vector.map(Number);
|
|
323
|
+
}
|
|
324
|
+
|
|
252
325
|
async function embed(text, options = {}) {
|
|
253
326
|
if (process.env.THUMBGATE_VECTOR_STUB_EMBED === 'true') {
|
|
254
327
|
// Deterministic 384-dim unit vector: first element = 1.0, rest = 0.0
|
|
@@ -277,6 +350,28 @@ async function embed(text, options = {}) {
|
|
|
277
350
|
console.warn(`Core AI embedding failed, falling back to local: ${coreaiError.message}`);
|
|
278
351
|
}
|
|
279
352
|
}
|
|
353
|
+
const ollamaConfig = getOllamaEmbeddingConfig();
|
|
354
|
+
if (ollamaConfig.enabled) {
|
|
355
|
+
try {
|
|
356
|
+
const vector = await embedWithOllama(text, options);
|
|
357
|
+
_lastEmbeddingProfile = {
|
|
358
|
+
generatedAt: new Date().toISOString(),
|
|
359
|
+
source: 'local-ollama',
|
|
360
|
+
activeProfile: {
|
|
361
|
+
id: 'ollama',
|
|
362
|
+
model: ollamaConfig.model,
|
|
363
|
+
outputDimensionality: vector.length,
|
|
364
|
+
task: options.task || 'code retrieval',
|
|
365
|
+
rationale: 'Explicit local Ollama semantic embedding provider.',
|
|
366
|
+
qualityTier: 'production',
|
|
367
|
+
},
|
|
368
|
+
fallbackUsed: false,
|
|
369
|
+
};
|
|
370
|
+
return vector;
|
|
371
|
+
} catch (ollamaError) {
|
|
372
|
+
console.warn(`Ollama embedding failed, falling back: ${ollamaError.message}`);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
280
375
|
if (geminiConfig.enabled) {
|
|
281
376
|
try {
|
|
282
377
|
const vector = await embedWithGemini(text, options);
|
|
@@ -297,7 +392,9 @@ async function embed(text, options = {}) {
|
|
|
297
392
|
if (!geminiConfig.fallbackToLocal) {
|
|
298
393
|
throw geminiError;
|
|
299
394
|
}
|
|
300
|
-
|
|
395
|
+
// Do not log raw provider/user-controlled error text (Sonar jssecurity:S5145).
|
|
396
|
+
const code = geminiError && (geminiError.code || geminiError.name || 'Error');
|
|
397
|
+
console.warn(`Gemini embedding fallback: ${code}`);
|
|
301
398
|
}
|
|
302
399
|
}
|
|
303
400
|
if (hasLocalTransformerProvider()) {
|
|
@@ -314,6 +411,8 @@ async function embed(text, options = {}) {
|
|
|
314
411
|
}
|
|
315
412
|
|
|
316
413
|
const vector = embedWithFeatureHash(text);
|
|
414
|
+
// Feature-hash is a last-resort degrade, not production semantic quality.
|
|
415
|
+
// Callers (prove/eval/chat health) must treat quality_tier=degraded.
|
|
317
416
|
_lastEmbeddingProfile = {
|
|
318
417
|
generatedAt: new Date().toISOString(),
|
|
319
418
|
source: 'built-in',
|
|
@@ -322,9 +421,11 @@ async function embed(text, options = {}) {
|
|
|
322
421
|
model: 'ThumbGate feature hashing',
|
|
323
422
|
outputDimensionality: FEATURE_HASH_DIMENSIONS,
|
|
324
423
|
task: options.task || 'code retrieval',
|
|
325
|
-
rationale: '
|
|
424
|
+
rationale: 'DEGRADED: deterministic zero-dependency hash embedding — not semantic. Configure Gemini or local transformers for production retrieval quality.',
|
|
425
|
+
qualityTier: 'degraded',
|
|
326
426
|
},
|
|
327
|
-
fallbackUsed:
|
|
427
|
+
fallbackUsed: true,
|
|
428
|
+
fallbackReason: 'no_managed_or_transformer_embedder',
|
|
328
429
|
};
|
|
329
430
|
return vector;
|
|
330
431
|
}
|
|
@@ -440,4 +541,7 @@ module.exports = {
|
|
|
440
541
|
setGeminiEmbedderForTests,
|
|
441
542
|
truncateForEmbedding,
|
|
442
543
|
embedWithFeatureHash,
|
|
544
|
+
embedWithOllama,
|
|
545
|
+
getOllamaEmbeddingConfig,
|
|
546
|
+
hasSemanticEmbeddingProvider,
|
|
443
547
|
};
|