thumbgate 1.2.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/README.md +4 -4
- package/.claude-plugin/marketplace.json +32 -13
- package/.claude-plugin/plugin.json +15 -2
- package/.well-known/llms.txt +60 -0
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +133 -23
- package/adapters/README.md +1 -1
- package/adapters/chatgpt/openapi.yaml +168 -0
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/codex/config.toml +2 -2
- package/adapters/mcp/server-stdio.js +85 -2
- package/adapters/opencode/opencode.json +1 -1
- package/bin/cli.js +215 -19
- package/bin/postinstall.js +8 -2
- package/config/budget.json +18 -0
- package/config/gates/code-edit.json +61 -0
- package/config/gates/db-write.json +61 -0
- package/config/gates/default.json +154 -3
- package/config/gates/deploy.json +61 -0
- package/config/github-about.json +2 -1
- package/config/merge-quality-checks.json +23 -0
- package/config/model-tiers.json +11 -0
- package/openapi/openapi.yaml +168 -0
- package/package.json +47 -13
- package/plugins/claude-codex-bridge/.claude-plugin/plugin.json +1 -1
- package/plugins/claude-codex-bridge/.mcp.json +1 -1
- package/plugins/claude-codex-bridge/scripts/codex-bridge.js +1 -3
- package/plugins/codex-profile/.codex-plugin/plugin.json +1 -1
- package/plugins/codex-profile/.mcp.json +1 -1
- package/plugins/codex-profile/INSTALL.md +27 -4
- package/plugins/codex-profile/README.md +33 -9
- package/plugins/cursor-marketplace/.cursor-plugin/plugin.json +1 -1
- package/plugins/cursor-marketplace/README.md +2 -2
- package/plugins/cursor-marketplace/commands/capture-feedback.md +2 -2
- package/plugins/cursor-marketplace/rules/feedback-capture.mdc +3 -3
- package/plugins/cursor-marketplace/skills/capture-feedback/SKILL.md +3 -2
- package/plugins/opencode-profile/INSTALL.md +1 -1
- package/public/blog.html +73 -0
- package/public/compare/mem0.html +189 -0
- package/public/compare/speclock.html +180 -0
- package/public/compare.html +12 -4
- package/public/guide.html +5 -5
- package/public/guides/claude-code-prevent-repeated-mistakes.html +161 -0
- package/public/guides/codex-cli-guardrails.html +158 -0
- package/public/guides/cursor-prevent-repeated-mistakes.html +161 -0
- package/public/guides/pre-action-gates.html +162 -0
- package/public/guides/stop-repeated-ai-agent-mistakes.html +159 -0
- package/public/index.html +169 -70
- package/public/learn/ai-agent-persistent-memory.html +1 -0
- package/public/lessons.html +334 -17
- package/public/llm-context.md +140 -0
- package/public/pro.html +24 -22
- package/scripts/__pycache__/train_from_feedback.cpython-312.pyc +0 -0
- package/scripts/access-anomaly-detector.js +1 -1
- package/scripts/adk-consolidator.js +1 -5
- package/scripts/agent-security-hardening.js +4 -6
- package/scripts/agentic-data-pipeline.js +1 -3
- package/scripts/async-job-runner.js +1 -5
- package/scripts/audit-trail.js +7 -5
- package/scripts/background-agent-governance.js +2 -10
- package/scripts/billing.js +2 -16
- package/scripts/budget-enforcer.js +173 -0
- package/scripts/build-codex-plugin.js +152 -0
- package/scripts/capture-railway-diagnostics.sh +97 -0
- package/scripts/check-congruence.js +133 -15
- package/scripts/claude-feedback-sync.js +320 -0
- package/scripts/cli-telemetry.js +4 -1
- package/scripts/commercial-offer.js +5 -7
- package/scripts/content-engine/linkedin-content-generator.js +154 -0
- package/scripts/content-engine/output/linkedin-memento-validation.md +17 -0
- package/scripts/content-engine/output/linkedin-posts-2026-04-09.md +175 -0
- package/scripts/content-engine/reddit-thread-finder.js +154 -0
- package/scripts/context-engine.js +21 -6
- package/scripts/contextfs.js +33 -44
- package/scripts/dashboard.js +104 -0
- package/scripts/decision-journal.js +341 -0
- package/scripts/delegation-runtime.js +1 -5
- package/scripts/distribution-surfaces.js +26 -0
- package/scripts/document-intake.js +927 -0
- package/scripts/ephemeral-agent-store.js +1 -8
- package/scripts/evolution-state.js +1 -5
- package/scripts/experiment-tracker.js +1 -5
- package/scripts/export-databricks-bundle.js +1 -5
- package/scripts/export-hf-dataset.js +1 -5
- package/scripts/export-training.js +1 -5
- package/scripts/feedback-attribution.js +1 -16
- package/scripts/feedback-history-distiller.js +1 -16
- package/scripts/feedback-loop.js +17 -5
- package/scripts/feedback-root-consolidator.js +2 -21
- package/scripts/feedback-session.js +49 -0
- package/scripts/feedback-to-rules.js +188 -28
- package/scripts/filesystem-search.js +1 -9
- package/scripts/fs-utils.js +104 -0
- package/scripts/gates-engine.js +149 -4
- package/scripts/github-about.js +32 -8
- package/scripts/gtm-revenue-loop.js +1 -5
- package/scripts/harness-selector.js +148 -0
- package/scripts/hosted-job-launcher.js +1 -5
- package/scripts/hybrid-feedback-context.js +7 -33
- package/scripts/intervention-policy.js +753 -0
- package/scripts/lesson-db.js +3 -18
- package/scripts/lesson-inference.js +194 -16
- package/scripts/lesson-retrieval.js +60 -24
- package/scripts/llm-client.js +59 -0
- package/scripts/local-model-profile.js +18 -2
- package/scripts/managed-lesson-agent.js +183 -0
- package/scripts/marketing-experiment.js +8 -22
- package/scripts/meta-agent-loop.js +624 -0
- package/scripts/metered-billing.js +1 -1
- package/scripts/model-tier-router.js +10 -1
- package/scripts/money-watcher.js +1 -4
- package/scripts/obsidian-export.js +1 -5
- package/scripts/operational-integrity.js +369 -34
- package/scripts/org-dashboard.js +6 -1
- package/scripts/per-step-scoring.js +2 -4
- package/scripts/pr-manager.js +201 -19
- package/scripts/pro-features.js +3 -2
- package/scripts/prompt-dlp.js +3 -3
- package/scripts/prove-adapters.js +2 -5
- package/scripts/prove-attribution.js +1 -5
- package/scripts/prove-automation.js +3 -5
- package/scripts/prove-cloudflare-sandbox.js +1 -3
- package/scripts/prove-data-pipeline.js +1 -3
- package/scripts/prove-intelligence.js +1 -3
- package/scripts/prove-lancedb.js +1 -5
- package/scripts/prove-local-intelligence.js +1 -3
- package/scripts/prove-packaged-runtime.js +326 -0
- package/scripts/prove-predictive-insights.js +1 -3
- package/scripts/prove-runtime.js +13 -0
- package/scripts/prove-training-export.js +1 -3
- package/scripts/prove-workflow-contract.js +1 -5
- package/scripts/rate-limiter.js +6 -4
- package/scripts/reddit-dm-outreach.js +14 -4
- package/scripts/schedule-manager.js +3 -5
- package/scripts/security-scanner.js +448 -0
- package/scripts/self-distill-agent.js +579 -0
- package/scripts/semantic-dedup.js +115 -0
- package/scripts/skill-exporter.js +1 -3
- package/scripts/skill-generator.js +1 -5
- package/scripts/social-analytics/engagement-audit.js +1 -18
- package/scripts/social-analytics/pollers/linkedin.js +26 -16
- package/scripts/social-analytics/publishers/linkedin.js +1 -1
- package/scripts/social-analytics/publishers/zernio.js +51 -0
- package/scripts/social-pipeline.js +1 -3
- package/scripts/social-post-hourly.js +47 -4
- package/scripts/statusline-links.js +6 -5
- package/scripts/statusline-local-stats.js +2 -0
- package/scripts/statusline.sh +38 -7
- package/scripts/sync-branch-protection.js +340 -0
- package/scripts/tessl-export.js +1 -3
- package/scripts/thumbgate-search.js +32 -1
- package/scripts/tool-kpi-tracker.js +1 -1
- package/scripts/tool-registry.js +108 -4
- package/scripts/vector-store.js +1 -5
- package/scripts/weekly-auto-post.js +1 -1
- package/scripts/workflow-sentinel.js +205 -4
- package/skills/thumbgate/SKILL.md +2 -2
- package/src/api/server.js +273 -4
- package/scripts/social-analytics/db/social-analytics.db-shm +0 -0
- /package/scripts/social-analytics/db/{social-analytics.db-wal → analytics.sqlite} +0 -0
package/scripts/dashboard.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const path = require('path');
|
|
6
6
|
const { aggregateFailureDiagnostics } = require('./failure-diagnostics');
|
|
7
|
+
const { AUDIT_LOG_FILENAME } = require('./audit-trail');
|
|
7
8
|
const { getBillingSummary, loadFunnelLedger, loadResolvedRevenueEvents } = require('./billing');
|
|
8
9
|
const { getTelemetryAnalytics, loadTelemetryEvents } = require('./telemetry-analytics');
|
|
9
10
|
const { getAutoGatesPath } = require('./auto-promote-gates');
|
|
@@ -19,6 +20,8 @@ const { routeProfile } = require('./profile-router');
|
|
|
19
20
|
const { getSettingsStatus } = require('./settings-hierarchy');
|
|
20
21
|
const { summarizeWorkflowRuns } = require('./workflow-runs');
|
|
21
22
|
const { searchLessons } = require('./lesson-search');
|
|
23
|
+
const { getInterventionPolicySummary } = require('./intervention-policy');
|
|
24
|
+
const { computeDecisionMetrics } = require('./decision-journal');
|
|
22
25
|
|
|
23
26
|
const PROJECT_ROOT = path.join(__dirname, '..');
|
|
24
27
|
const DEFAULT_GATES_PATH = path.join(PROJECT_ROOT, 'config', 'gates', 'default.json');
|
|
@@ -60,6 +63,15 @@ function pickFirstText(...values) {
|
|
|
60
63
|
return null;
|
|
61
64
|
}
|
|
62
65
|
|
|
66
|
+
function toLocalDayKey(value) {
|
|
67
|
+
const ts = value instanceof Date ? value : new Date(value);
|
|
68
|
+
if (Number.isNaN(ts.getTime())) return null;
|
|
69
|
+
const year = ts.getFullYear();
|
|
70
|
+
const month = String(ts.getMonth() + 1).padStart(2, '0');
|
|
71
|
+
const day = String(ts.getDate()).padStart(2, '0');
|
|
72
|
+
return `${year}-${month}-${day}`;
|
|
73
|
+
}
|
|
74
|
+
|
|
63
75
|
// ---------------------------------------------------------------------------
|
|
64
76
|
// Approval rate + trend
|
|
65
77
|
// ---------------------------------------------------------------------------
|
|
@@ -143,6 +155,58 @@ function computeGateStats() {
|
|
|
143
155
|
};
|
|
144
156
|
}
|
|
145
157
|
|
|
158
|
+
function computeGateAuditSeries(feedbackDir, options = {}) {
|
|
159
|
+
const auditLogPath = path.join(feedbackDir, AUDIT_LOG_FILENAME);
|
|
160
|
+
const entries = readJSONL(auditLogPath).filter((entry) => entry && entry.timestamp);
|
|
161
|
+
const dayCount = Number.isInteger(options.dayCount) ? options.dayCount : 14;
|
|
162
|
+
const today = new Date();
|
|
163
|
+
today.setHours(0, 0, 0, 0);
|
|
164
|
+
const countsByDay = new Map();
|
|
165
|
+
|
|
166
|
+
for (const entry of entries) {
|
|
167
|
+
if (!['allow', 'deny', 'warn'].includes(entry.decision)) continue;
|
|
168
|
+
const dayKey = toLocalDayKey(entry.timestamp);
|
|
169
|
+
if (!dayKey) continue;
|
|
170
|
+
if (!countsByDay.has(dayKey)) {
|
|
171
|
+
countsByDay.set(dayKey, { allow: 0, deny: 0, warn: 0 });
|
|
172
|
+
}
|
|
173
|
+
countsByDay.get(dayKey)[entry.decision] += 1;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const days = [];
|
|
177
|
+
const totals = { allow: 0, deny: 0, warn: 0, intercepted: 0, total: 0 };
|
|
178
|
+
|
|
179
|
+
for (let offset = dayCount - 1; offset >= 0; offset -= 1) {
|
|
180
|
+
const day = new Date(today);
|
|
181
|
+
day.setDate(today.getDate() - offset);
|
|
182
|
+
const dayKey = toLocalDayKey(day);
|
|
183
|
+
const record = countsByDay.get(dayKey) || { allow: 0, deny: 0, warn: 0 };
|
|
184
|
+
const intercepted = record.deny + record.warn;
|
|
185
|
+
const total = intercepted + record.allow;
|
|
186
|
+
const summary = {
|
|
187
|
+
dayKey,
|
|
188
|
+
allow: record.allow,
|
|
189
|
+
deny: record.deny,
|
|
190
|
+
warn: record.warn,
|
|
191
|
+
intercepted,
|
|
192
|
+
total,
|
|
193
|
+
};
|
|
194
|
+
totals.allow += record.allow;
|
|
195
|
+
totals.deny += record.deny;
|
|
196
|
+
totals.warn += record.warn;
|
|
197
|
+
totals.intercepted += intercepted;
|
|
198
|
+
totals.total += total;
|
|
199
|
+
days.push(summary);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
dayCount,
|
|
204
|
+
days,
|
|
205
|
+
totals,
|
|
206
|
+
activeDays: days.filter((day) => day.total > 0).length,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
146
210
|
function listActiveGates() {
|
|
147
211
|
try {
|
|
148
212
|
const config = loadGatesConfig();
|
|
@@ -710,6 +774,7 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
710
774
|
const prevention = computePreventionImpact(feedbackDir, gateStats);
|
|
711
775
|
const trend = computeSessionTrend(entries, 10);
|
|
712
776
|
const health = computeSystemHealth(feedbackDir, gateStats);
|
|
777
|
+
const gateAudit = computeGateAuditSeries(feedbackDir);
|
|
713
778
|
const diagnostics = aggregateFailureDiagnostics([...entries, ...diagnosticEntries]);
|
|
714
779
|
const secretGuard = computeSecretGuardStats(diagnosticEntries);
|
|
715
780
|
const gates = listActiveGates();
|
|
@@ -722,6 +787,8 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
722
787
|
const delegation = summarizeDelegation(feedbackDir);
|
|
723
788
|
const readiness = generateAgentReadinessReport({ projectRoot: PROJECT_ROOT });
|
|
724
789
|
const harness = computeHarnessOverview(feedbackDir, entries);
|
|
790
|
+
const interventionPolicy = getInterventionPolicySummary(feedbackDir);
|
|
791
|
+
const decisions = computeDecisionMetrics(feedbackDir);
|
|
725
792
|
const settingsStatus = getSettingsStatus({ projectRoot: PROJECT_ROOT });
|
|
726
793
|
settingsStatus.routingPreview = {
|
|
727
794
|
dashboardTool: routeProfile({
|
|
@@ -755,6 +822,13 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
755
822
|
lessonEffectiveness: { rate: totalNeg > 0 ? Math.round((autoGates / totalNeg) * 10000) / 100 : 0, totalNegative: totalNeg, autoGatesCreated: autoGates },
|
|
756
823
|
errorTrend: { direction: lastWeekNeg > 0 ? (negRecent.length < lastWeekNeg ? 'improving' : negRecent.length > lastWeekNeg ? 'worsening' : 'stable') : (negRecent.length > 0 ? 'new-errors' : 'clean'), thisWeek: negRecent.length, lastWeek: lastWeekNeg },
|
|
757
824
|
weeklyActivity: { positive: posRecent.length, negative: negRecent.length, total: recentEntries.length },
|
|
825
|
+
decisionLoop: {
|
|
826
|
+
fastPathRate: decisions.fastPathRate,
|
|
827
|
+
overrideRate: decisions.overrideRate,
|
|
828
|
+
rollbackRate: decisions.rollbackRate,
|
|
829
|
+
medianLatencyMs: decisions.medianLatencyMs,
|
|
830
|
+
resolvedCount: decisions.resolvedCount,
|
|
831
|
+
},
|
|
758
832
|
};
|
|
759
833
|
|
|
760
834
|
const team = generateOrgDashboard({
|
|
@@ -782,6 +856,7 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
782
856
|
prevention,
|
|
783
857
|
trend,
|
|
784
858
|
health,
|
|
859
|
+
gateAudit,
|
|
785
860
|
diagnostics,
|
|
786
861
|
delegation,
|
|
787
862
|
secretGuard,
|
|
@@ -790,6 +865,8 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
790
865
|
observability,
|
|
791
866
|
instrumentation,
|
|
792
867
|
readiness,
|
|
868
|
+
interventionPolicy,
|
|
869
|
+
decisions,
|
|
793
870
|
settingsStatus,
|
|
794
871
|
team,
|
|
795
872
|
templateLibrary,
|
|
@@ -809,6 +886,7 @@ function printDashboard(data) {
|
|
|
809
886
|
prevention,
|
|
810
887
|
trend,
|
|
811
888
|
health,
|
|
889
|
+
gateAudit,
|
|
812
890
|
diagnostics,
|
|
813
891
|
delegation,
|
|
814
892
|
secretGuard,
|
|
@@ -817,6 +895,8 @@ function printDashboard(data) {
|
|
|
817
895
|
observability,
|
|
818
896
|
instrumentation,
|
|
819
897
|
readiness,
|
|
898
|
+
interventionPolicy,
|
|
899
|
+
decisions,
|
|
820
900
|
settingsStatus,
|
|
821
901
|
team,
|
|
822
902
|
templateLibrary,
|
|
@@ -862,6 +942,28 @@ function printDashboard(data) {
|
|
|
862
942
|
console.log(` Top Next Fix : ${harness.topRecommendations[0].type} (${harness.topRecommendations[0].count} lessons)`);
|
|
863
943
|
}
|
|
864
944
|
|
|
945
|
+
console.log('');
|
|
946
|
+
console.log('🧠 Learned Policy');
|
|
947
|
+
console.log(` Enabled : ${interventionPolicy.enabled ? 'yes' : 'no'}`);
|
|
948
|
+
console.log(` Examples : ${interventionPolicy.exampleCount}`);
|
|
949
|
+
console.log(` Train Accuracy : ${Math.round((interventionPolicy.metrics.trainingAccuracy || 0) * 100)}%`);
|
|
950
|
+
console.log(` Holdout Accuracy : ${Math.round((interventionPolicy.metrics.holdoutAccuracy || 0) * 100)}%`);
|
|
951
|
+
console.log(` Recent Pressure : ${Math.round((interventionPolicy.nonAllowRate || 0) * 100)}% non-allow`);
|
|
952
|
+
if (interventionPolicy.updatedAt) {
|
|
953
|
+
console.log(` Updated : ${interventionPolicy.updatedAt}`);
|
|
954
|
+
}
|
|
955
|
+
if (interventionPolicy.topTokens && interventionPolicy.topTokens.deny && interventionPolicy.topTokens.deny[0]) {
|
|
956
|
+
console.log(` Top Deny Signal : ${interventionPolicy.topTokens.deny[0].token}`);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
console.log('');
|
|
960
|
+
console.log('🧭 Decision Loop');
|
|
961
|
+
console.log(` Evaluations : ${decisions.evaluationCount}`);
|
|
962
|
+
console.log(` Fast Path : ${Math.round((decisions.fastPathRate || 0) * 100)}%`);
|
|
963
|
+
console.log(` Override Rate : ${Math.round((decisions.overrideRate || 0) * 100)}%`);
|
|
964
|
+
console.log(` Rollback Rate : ${Math.round((decisions.rollbackRate || 0) * 100)}%`);
|
|
965
|
+
console.log(` Median Latency : ${Math.round((decisions.medianLatencyMs || 0) / 1000)}s`);
|
|
966
|
+
|
|
865
967
|
console.log('');
|
|
866
968
|
console.log('🎯 North Star');
|
|
867
969
|
console.log(` Weekly Proof Runs: ${analytics.northStar.weeklyActiveProofBackedWorkflowRuns}`);
|
|
@@ -1037,12 +1139,14 @@ module.exports = {
|
|
|
1037
1139
|
generateDashboard,
|
|
1038
1140
|
printDashboard,
|
|
1039
1141
|
computeApprovalStats,
|
|
1142
|
+
computeDecisionMetrics,
|
|
1040
1143
|
computeGateStats,
|
|
1041
1144
|
computePreventionImpact,
|
|
1042
1145
|
computeSessionTrend,
|
|
1043
1146
|
computeSystemHealth,
|
|
1044
1147
|
computeEfficiencyMetrics,
|
|
1045
1148
|
computeHarnessOverview,
|
|
1149
|
+
getInterventionPolicySummary,
|
|
1046
1150
|
computeAnalyticsSummary,
|
|
1047
1151
|
computeSecretGuardStats,
|
|
1048
1152
|
computeObservabilityStats,
|
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
const fs = require('fs');
|
|
6
|
+
const path = require('path');
|
|
7
|
+
const { resolveFeedbackDir } = require('./feedback-paths');
|
|
8
|
+
const { sanitizeToolInput } = require('./audit-trail');
|
|
9
|
+
const { ensureDir } = require('./fs-utils');
|
|
10
|
+
|
|
11
|
+
const DECISION_LOG_FILENAME = 'decision-journal.jsonl';
|
|
12
|
+
const DEFAULT_DAY_COUNT = 14;
|
|
13
|
+
const RESOLVED_OUTCOMES = new Set(['accepted', 'completed', 'overridden', 'rolled_back', 'blocked', 'aborted']);
|
|
14
|
+
const DECISION_OUTCOMES = new Set([...RESOLVED_OUTCOMES, 'warned']);
|
|
15
|
+
|
|
16
|
+
function getDecisionLogPath(feedbackDir) {
|
|
17
|
+
return path.join(resolveFeedbackDir({ feedbackDir }), DECISION_LOG_FILENAME);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
function buildActionId(prefix = 'decision') {
|
|
22
|
+
return `${prefix}_${Date.now()}_${crypto.randomBytes(3).toString('hex')}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function readDecisionLog(logPath) {
|
|
26
|
+
const targetPath = logPath || getDecisionLogPath();
|
|
27
|
+
if (!fs.existsSync(targetPath)) return [];
|
|
28
|
+
const raw = fs.readFileSync(targetPath, 'utf8').trim();
|
|
29
|
+
if (!raw) return [];
|
|
30
|
+
return raw
|
|
31
|
+
.split('\n')
|
|
32
|
+
.map((line) => {
|
|
33
|
+
try {
|
|
34
|
+
return JSON.parse(line);
|
|
35
|
+
} catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
})
|
|
39
|
+
.filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function appendDecisionRecord(record, feedbackDir) {
|
|
43
|
+
const logPath = getDecisionLogPath(feedbackDir);
|
|
44
|
+
ensureDir(path.dirname(logPath));
|
|
45
|
+
fs.appendFileSync(logPath, `${JSON.stringify(record)}\n`, 'utf8');
|
|
46
|
+
return record;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function toLocalDayKey(value) {
|
|
50
|
+
const ts = value instanceof Date ? value : new Date(value);
|
|
51
|
+
if (Number.isNaN(ts.getTime())) return null;
|
|
52
|
+
const year = ts.getFullYear();
|
|
53
|
+
const month = String(ts.getMonth() + 1).padStart(2, '0');
|
|
54
|
+
const day = String(ts.getDate()).padStart(2, '0');
|
|
55
|
+
return `${year}-${month}-${day}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function normalizeOutcome(value) {
|
|
59
|
+
const normalized = String(value || '').trim().toLowerCase().replace(/[\s-]+/g, '_');
|
|
60
|
+
if (DECISION_OUTCOMES.has(normalized)) return normalized;
|
|
61
|
+
return 'completed';
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function inferActualDecision(outcome, fallback) {
|
|
65
|
+
if (fallback) return String(fallback);
|
|
66
|
+
if (outcome === 'blocked') return 'deny';
|
|
67
|
+
if (outcome === 'warned') return 'warn';
|
|
68
|
+
return 'allow';
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function median(values) {
|
|
72
|
+
const sorted = values
|
|
73
|
+
.map((value) => Number(value))
|
|
74
|
+
.filter((value) => Number.isFinite(value))
|
|
75
|
+
.sort((left, right) => left - right);
|
|
76
|
+
if (sorted.length === 0) return 0;
|
|
77
|
+
const middle = Math.floor(sorted.length / 2);
|
|
78
|
+
if (sorted.length % 2 === 1) return sorted[middle];
|
|
79
|
+
return Math.round((sorted[middle - 1] + sorted[middle]) / 2);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function summarizeBlastRadius(report = {}) {
|
|
83
|
+
const blastRadius = report.blastRadius || {};
|
|
84
|
+
return {
|
|
85
|
+
severity: blastRadius.severity || 'low',
|
|
86
|
+
fileCount: Number(blastRadius.fileCount || 0),
|
|
87
|
+
surfaceCount: Number(blastRadius.surfaceCount || 0),
|
|
88
|
+
releaseSensitiveCount: Array.isArray(blastRadius.releaseSensitiveFiles) ? blastRadius.releaseSensitiveFiles.length : 0,
|
|
89
|
+
protectedWithoutApprovalCount: Array.isArray(blastRadius.unapprovedProtectedFiles) ? blastRadius.unapprovedProtectedFiles.length : 0,
|
|
90
|
+
summary: blastRadius.summary || '',
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function normalizeRecommendation(report = {}) {
|
|
95
|
+
const control = report.decisionControl || {};
|
|
96
|
+
return {
|
|
97
|
+
decision: report.decision || 'allow',
|
|
98
|
+
riskScore: Number(report.riskScore || 0),
|
|
99
|
+
riskBand: report.band || 'low',
|
|
100
|
+
executionMode: control.executionMode || (report.decision === 'deny' ? 'blocked' : report.decision === 'warn' ? 'checkpoint_required' : 'auto_execute'),
|
|
101
|
+
decisionOwner: control.decisionOwner || (report.decision === 'allow' ? 'agent' : 'shared'),
|
|
102
|
+
reversibility: control.reversibility || 'reviewable',
|
|
103
|
+
requiresHumanApproval: control.requiresHumanApproval === true,
|
|
104
|
+
summary: report.summary || '',
|
|
105
|
+
recommendedAction: control.recommendedAction || (report.decision === 'deny' ? 'halt' : report.decision === 'warn' ? 'review' : 'proceed'),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function recordDecisionEvaluation(report, params = {}, options = {}) {
|
|
110
|
+
const actionId = params.actionId || buildActionId();
|
|
111
|
+
const changedFiles = Array.isArray(params.changedFiles)
|
|
112
|
+
? params.changedFiles.slice()
|
|
113
|
+
: Array.isArray(report && report.blastRadius && report.blastRadius.affectedFiles)
|
|
114
|
+
? report.blastRadius.affectedFiles.slice()
|
|
115
|
+
: [];
|
|
116
|
+
const record = {
|
|
117
|
+
recordType: 'evaluation',
|
|
118
|
+
actionId,
|
|
119
|
+
timestamp: params.timestamp || new Date().toISOString(),
|
|
120
|
+
source: params.source || 'workflow-sentinel',
|
|
121
|
+
toolName: params.toolName || report.toolName || 'unknown',
|
|
122
|
+
toolInput: sanitizeToolInput(params.toolInput || {}),
|
|
123
|
+
changedFiles,
|
|
124
|
+
recommendation: normalizeRecommendation(report),
|
|
125
|
+
blastRadius: summarizeBlastRadius(report),
|
|
126
|
+
learnedPolicy: report.learnedPolicy && report.learnedPolicy.enabled
|
|
127
|
+
? {
|
|
128
|
+
label: report.learnedPolicy.prediction && report.learnedPolicy.prediction.label || null,
|
|
129
|
+
confidence: Number((report.learnedPolicy.prediction && report.learnedPolicy.prediction.confidence) || 0),
|
|
130
|
+
}
|
|
131
|
+
: null,
|
|
132
|
+
topRemediations: Array.isArray(report.remediations)
|
|
133
|
+
? report.remediations.slice(0, 3).map((entry) => ({ id: entry.id, title: entry.title }))
|
|
134
|
+
: [],
|
|
135
|
+
evidence: Array.isArray(report.evidence) ? report.evidence.slice(0, 4) : [],
|
|
136
|
+
};
|
|
137
|
+
return appendDecisionRecord(record, options.feedbackDir);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function recordDecisionOutcome(params = {}, options = {}) {
|
|
141
|
+
const actionId = params.actionId || buildActionId('decision_outcome');
|
|
142
|
+
const entries = readDecisionLog(getDecisionLogPath(options.feedbackDir));
|
|
143
|
+
const evaluation = [...entries]
|
|
144
|
+
.reverse()
|
|
145
|
+
.find((entry) => entry && entry.recordType === 'evaluation' && entry.actionId === actionId) || null;
|
|
146
|
+
const outcome = normalizeOutcome(params.outcome);
|
|
147
|
+
const timestamp = params.timestamp || new Date().toISOString();
|
|
148
|
+
const latencyMs = Number.isFinite(params.latencyMs)
|
|
149
|
+
? Number(params.latencyMs)
|
|
150
|
+
: evaluation && evaluation.timestamp
|
|
151
|
+
? Math.max(0, new Date(timestamp).getTime() - new Date(evaluation.timestamp).getTime())
|
|
152
|
+
: null;
|
|
153
|
+
const record = {
|
|
154
|
+
recordType: 'outcome',
|
|
155
|
+
actionId,
|
|
156
|
+
timestamp,
|
|
157
|
+
source: params.source || 'api',
|
|
158
|
+
actor: params.actor || 'human',
|
|
159
|
+
outcome,
|
|
160
|
+
actualDecision: inferActualDecision(outcome, params.actualDecision),
|
|
161
|
+
notes: params.notes || '',
|
|
162
|
+
metadata: params.metadata && typeof params.metadata === 'object' ? params.metadata : {},
|
|
163
|
+
latencyMs: Number.isFinite(latencyMs) ? latencyMs : null,
|
|
164
|
+
recommendation: evaluation ? evaluation.recommendation : (params.recommendation || null),
|
|
165
|
+
toolName: evaluation ? evaluation.toolName : (params.toolName || 'unknown'),
|
|
166
|
+
changedFiles: evaluation ? evaluation.changedFiles : (Array.isArray(params.changedFiles) ? params.changedFiles.slice() : []),
|
|
167
|
+
};
|
|
168
|
+
return appendDecisionRecord(record, options.feedbackDir);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function collapseDecisionTimeline(records) {
|
|
172
|
+
const actions = new Map();
|
|
173
|
+
for (const record of records) {
|
|
174
|
+
if (!record || !record.actionId) continue;
|
|
175
|
+
if (!actions.has(record.actionId)) {
|
|
176
|
+
actions.set(record.actionId, { actionId: record.actionId, evaluation: null, outcomes: [] });
|
|
177
|
+
}
|
|
178
|
+
const bucket = actions.get(record.actionId);
|
|
179
|
+
if (record.recordType === 'evaluation') {
|
|
180
|
+
bucket.evaluation = record;
|
|
181
|
+
} else if (record.recordType === 'outcome') {
|
|
182
|
+
bucket.outcomes.push(record);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
for (const bucket of actions.values()) {
|
|
186
|
+
bucket.outcomes.sort((left, right) => new Date(left.timestamp).getTime() - new Date(right.timestamp).getTime());
|
|
187
|
+
}
|
|
188
|
+
return [...actions.values()].sort((left, right) => {
|
|
189
|
+
const leftTs = left.evaluation ? new Date(left.evaluation.timestamp).getTime() : 0;
|
|
190
|
+
const rightTs = right.evaluation ? new Date(right.evaluation.timestamp).getTime() : 0;
|
|
191
|
+
return leftTs - rightTs;
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function initializeDaySeries(dayCount) {
|
|
196
|
+
const today = new Date();
|
|
197
|
+
today.setHours(0, 0, 0, 0);
|
|
198
|
+
const days = [];
|
|
199
|
+
for (let offset = dayCount - 1; offset >= 0; offset -= 1) {
|
|
200
|
+
const day = new Date(today);
|
|
201
|
+
day.setDate(today.getDate() - offset);
|
|
202
|
+
days.push({
|
|
203
|
+
dayKey: toLocalDayKey(day),
|
|
204
|
+
evaluations: 0,
|
|
205
|
+
fastPath: 0,
|
|
206
|
+
checkpoint: 0,
|
|
207
|
+
blockedRecommendations: 0,
|
|
208
|
+
overrides: 0,
|
|
209
|
+
rollbacks: 0,
|
|
210
|
+
completions: 0,
|
|
211
|
+
blockedOutcomes: 0,
|
|
212
|
+
latencies: [],
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
return days;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function safeRate(numerator, denominator) {
|
|
219
|
+
if (!denominator) return 0;
|
|
220
|
+
return Number((numerator / denominator).toFixed(4));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function computeDecisionMetrics(feedbackDir, options = {}) {
|
|
224
|
+
const dayCount = Number.isInteger(options.dayCount) ? options.dayCount : DEFAULT_DAY_COUNT;
|
|
225
|
+
const records = readDecisionLog(getDecisionLogPath(feedbackDir));
|
|
226
|
+
const actions = collapseDecisionTimeline(records).filter((entry) => entry.evaluation);
|
|
227
|
+
const series = initializeDaySeries(dayCount);
|
|
228
|
+
const dayMap = new Map(series.map((day) => [day.dayKey, day]));
|
|
229
|
+
const outcomeCounts = {
|
|
230
|
+
accepted: 0,
|
|
231
|
+
completed: 0,
|
|
232
|
+
overridden: 0,
|
|
233
|
+
rolled_back: 0,
|
|
234
|
+
blocked: 0,
|
|
235
|
+
aborted: 0,
|
|
236
|
+
warned: 0,
|
|
237
|
+
};
|
|
238
|
+
|
|
239
|
+
let fastPathCount = 0;
|
|
240
|
+
let checkpointCount = 0;
|
|
241
|
+
let blockedRecommendationCount = 0;
|
|
242
|
+
let overrideCount = 0;
|
|
243
|
+
let rollbackCount = 0;
|
|
244
|
+
let resolvedCount = 0;
|
|
245
|
+
const latencyValues = [];
|
|
246
|
+
|
|
247
|
+
for (const action of actions) {
|
|
248
|
+
const evaluation = action.evaluation;
|
|
249
|
+
const recommendation = evaluation.recommendation || {};
|
|
250
|
+
const evalDay = dayMap.get(toLocalDayKey(evaluation.timestamp));
|
|
251
|
+
if (evalDay) {
|
|
252
|
+
evalDay.evaluations += 1;
|
|
253
|
+
if (recommendation.executionMode === 'auto_execute') evalDay.fastPath += 1;
|
|
254
|
+
if (recommendation.executionMode === 'checkpoint_required') evalDay.checkpoint += 1;
|
|
255
|
+
if (recommendation.executionMode === 'blocked') evalDay.blockedRecommendations += 1;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
if (recommendation.executionMode === 'auto_execute') fastPathCount += 1;
|
|
259
|
+
if (recommendation.executionMode === 'checkpoint_required') checkpointCount += 1;
|
|
260
|
+
if (recommendation.executionMode === 'blocked') blockedRecommendationCount += 1;
|
|
261
|
+
|
|
262
|
+
const hasOverride = action.outcomes.some((outcome) => outcome.outcome === 'overridden');
|
|
263
|
+
const hasRollback = action.outcomes.some((outcome) => outcome.outcome === 'rolled_back');
|
|
264
|
+
if (hasOverride) overrideCount += 1;
|
|
265
|
+
if (hasRollback) rollbackCount += 1;
|
|
266
|
+
|
|
267
|
+
const latestOutcome = action.outcomes.length > 0 ? action.outcomes[action.outcomes.length - 1] : null;
|
|
268
|
+
if (latestOutcome) {
|
|
269
|
+
outcomeCounts[latestOutcome.outcome] = (outcomeCounts[latestOutcome.outcome] || 0) + 1;
|
|
270
|
+
const outcomeDay = dayMap.get(toLocalDayKey(latestOutcome.timestamp));
|
|
271
|
+
if (outcomeDay) {
|
|
272
|
+
if (latestOutcome.outcome === 'overridden') outcomeDay.overrides += 1;
|
|
273
|
+
if (latestOutcome.outcome === 'rolled_back') outcomeDay.rollbacks += 1;
|
|
274
|
+
if (latestOutcome.outcome === 'completed' || latestOutcome.outcome === 'accepted') outcomeDay.completions += 1;
|
|
275
|
+
if (latestOutcome.outcome === 'blocked') outcomeDay.blockedOutcomes += 1;
|
|
276
|
+
}
|
|
277
|
+
if (RESOLVED_OUTCOMES.has(latestOutcome.outcome)) {
|
|
278
|
+
resolvedCount += 1;
|
|
279
|
+
}
|
|
280
|
+
if (Number.isFinite(latestOutcome.latencyMs)) {
|
|
281
|
+
latencyValues.push(latestOutcome.latencyMs);
|
|
282
|
+
if (outcomeDay) outcomeDay.latencies.push(latestOutcome.latencyMs);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const days = series.map((day) => ({
|
|
288
|
+
dayKey: day.dayKey,
|
|
289
|
+
evaluations: day.evaluations,
|
|
290
|
+
fastPath: day.fastPath,
|
|
291
|
+
checkpoint: day.checkpoint,
|
|
292
|
+
blockedRecommendations: day.blockedRecommendations,
|
|
293
|
+
overrides: day.overrides,
|
|
294
|
+
rollbacks: day.rollbacks,
|
|
295
|
+
completions: day.completions,
|
|
296
|
+
blockedOutcomes: day.blockedOutcomes,
|
|
297
|
+
medianLatencyMs: median(day.latencies),
|
|
298
|
+
}));
|
|
299
|
+
|
|
300
|
+
return {
|
|
301
|
+
evaluationCount: actions.length,
|
|
302
|
+
resolvedCount,
|
|
303
|
+
fastPathCount,
|
|
304
|
+
checkpointCount,
|
|
305
|
+
blockedRecommendationCount,
|
|
306
|
+
overrideCount,
|
|
307
|
+
rollbackCount,
|
|
308
|
+
outcomeCounts,
|
|
309
|
+
fastPathRate: safeRate(fastPathCount, actions.length),
|
|
310
|
+
checkpointRate: safeRate(checkpointCount, actions.length),
|
|
311
|
+
overrideRate: safeRate(overrideCount, resolvedCount || actions.length),
|
|
312
|
+
rollbackRate: safeRate(rollbackCount, resolvedCount || actions.length),
|
|
313
|
+
followRate: safeRate(Math.max(0, resolvedCount - overrideCount), resolvedCount || actions.length),
|
|
314
|
+
medianLatencyMs: median(latencyValues),
|
|
315
|
+
averageLatencyMs: latencyValues.length > 0
|
|
316
|
+
? Math.round(latencyValues.reduce((sum, value) => sum + value, 0) / latencyValues.length)
|
|
317
|
+
: 0,
|
|
318
|
+
dayCount,
|
|
319
|
+
days,
|
|
320
|
+
activeDays: days.filter((day) => {
|
|
321
|
+
return day.evaluations > 0 || day.overrides > 0 || day.rollbacks > 0 || day.completions > 0 || day.blockedOutcomes > 0;
|
|
322
|
+
}).length,
|
|
323
|
+
};
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
module.exports = {
|
|
327
|
+
DECISION_LOG_FILENAME,
|
|
328
|
+
RESOLVED_OUTCOMES,
|
|
329
|
+
buildActionId,
|
|
330
|
+
collapseDecisionTimeline,
|
|
331
|
+
computeDecisionMetrics,
|
|
332
|
+
getDecisionLogPath,
|
|
333
|
+
normalizeOutcome,
|
|
334
|
+
readDecisionLog,
|
|
335
|
+
recordDecisionEvaluation,
|
|
336
|
+
recordDecisionOutcome,
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
if (require.main === module) {
|
|
340
|
+
console.log(JSON.stringify(computeDecisionMetrics(process.argv[2]), null, 2));
|
|
341
|
+
}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
const crypto = require('crypto');
|
|
5
5
|
const fs = require('fs');
|
|
6
6
|
const path = require('path');
|
|
7
|
+
const { ensureDir } = require('./fs-utils');
|
|
7
8
|
const {
|
|
8
9
|
loadSubagentProfiles,
|
|
9
10
|
getAllowedTools,
|
|
@@ -32,11 +33,6 @@ function getVerificationLoopModule() {
|
|
|
32
33
|
return require('./verification-loop');
|
|
33
34
|
}
|
|
34
35
|
|
|
35
|
-
function ensureDir(dirPath) {
|
|
36
|
-
if (!fs.existsSync(dirPath)) {
|
|
37
|
-
fs.mkdirSync(dirPath, { recursive: true });
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
36
|
|
|
41
37
|
function readJSONL(filePath) {
|
|
42
38
|
if (!fs.existsSync(filePath)) return [];
|
|
@@ -7,6 +7,8 @@ const ROOT = path.join(__dirname, '..');
|
|
|
7
7
|
const PRODUCTHUNT_URL = 'https://www.producthunt.com/products/thumbgate';
|
|
8
8
|
const CLAUDE_PLUGIN_LATEST_ASSET_NAME = 'thumbgate-claude-desktop.mcpb';
|
|
9
9
|
const CLAUDE_PLUGIN_NEXT_ASSET_NAME = 'thumbgate-claude-desktop-next.mcpb';
|
|
10
|
+
const CODEX_PLUGIN_LATEST_ASSET_NAME = 'thumbgate-codex-plugin.zip';
|
|
11
|
+
const CODEX_PLUGIN_NEXT_ASSET_NAME = 'thumbgate-codex-plugin-next.zip';
|
|
10
12
|
|
|
11
13
|
function readJson(root, relativePath) {
|
|
12
14
|
return JSON.parse(fs.readFileSync(path.join(root, relativePath), 'utf8'));
|
|
@@ -42,14 +44,38 @@ function getClaudePluginVersionedDownloadUrl(root = ROOT, version = getPackageVe
|
|
|
42
44
|
return `${getRepositoryUrl(root)}/releases/download/v${normalized}/${getClaudePluginVersionedAssetName(normalized)}`;
|
|
43
45
|
}
|
|
44
46
|
|
|
47
|
+
function getCodexPluginVersionedAssetName(version = getPackageVersion(ROOT)) {
|
|
48
|
+
const normalized = String(version || '').replace(/^v/, '');
|
|
49
|
+
return `thumbgate-codex-plugin-v${normalized}.zip`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function getCodexPluginChannelAssetName(version = getPackageVersion(ROOT)) {
|
|
53
|
+
return isPrereleaseVersion(version) ? CODEX_PLUGIN_NEXT_ASSET_NAME : CODEX_PLUGIN_LATEST_ASSET_NAME;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function getCodexPluginLatestDownloadUrl(root = ROOT) {
|
|
57
|
+
return `${getRepositoryUrl(root)}/releases/latest/download/${CODEX_PLUGIN_LATEST_ASSET_NAME}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function getCodexPluginVersionedDownloadUrl(root = ROOT, version = getPackageVersion(root)) {
|
|
61
|
+
const normalized = String(version || '').replace(/^v/, '');
|
|
62
|
+
return `${getRepositoryUrl(root)}/releases/download/v${normalized}/${getCodexPluginVersionedAssetName(normalized)}`;
|
|
63
|
+
}
|
|
64
|
+
|
|
45
65
|
module.exports = {
|
|
46
66
|
CLAUDE_PLUGIN_LATEST_ASSET_NAME,
|
|
47
67
|
CLAUDE_PLUGIN_NEXT_ASSET_NAME,
|
|
68
|
+
CODEX_PLUGIN_LATEST_ASSET_NAME,
|
|
69
|
+
CODEX_PLUGIN_NEXT_ASSET_NAME,
|
|
48
70
|
PRODUCTHUNT_URL,
|
|
49
71
|
getClaudePluginChannelAssetName,
|
|
50
72
|
getClaudePluginLatestDownloadUrl,
|
|
51
73
|
getClaudePluginVersionedAssetName,
|
|
52
74
|
getClaudePluginVersionedDownloadUrl,
|
|
75
|
+
getCodexPluginChannelAssetName,
|
|
76
|
+
getCodexPluginLatestDownloadUrl,
|
|
77
|
+
getCodexPluginVersionedAssetName,
|
|
78
|
+
getCodexPluginVersionedDownloadUrl,
|
|
53
79
|
getPackageVersion,
|
|
54
80
|
getRepositoryUrl,
|
|
55
81
|
isPrereleaseVersion,
|