thumbgate 1.34.3 → 1.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/cyberstrike-compare-not-clone/SKILL.md +36 -0
- package/.agents/skills/gitlab-sandbox-allowlist-not-trust/SKILL.md +77 -0
- package/.agents/skills/jit-harness-compare-not-clone/SKILL.md +34 -0
- package/.agents/skills/openui-catalog-compose-honesty/SKILL.md +64 -0
- package/.agents/skills/zvec-grep-compare-not-clone/SKILL.md +34 -0
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/llms.txt +1 -0
- package/.well-known/mcp/server-card.json +1 -1
- package/CONTRIBUTING.md +95 -0
- package/README.md +195 -632
- package/THIRD_PARTY_NOTICES.md +89 -0
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/future-agi/.mcp.json +8 -0
- package/adapters/future-agi/FUTURE_AGI.md +23 -0
- package/adapters/future-agi/config.toml +3 -0
- package/adapters/future-agi/future-agi-bridge.js +9 -0
- package/adapters/future-agi/opencode.json +8 -0
- package/adapters/herdr/herdr-plugin.toml +18 -0
- package/adapters/mcp/server-stdio.js +238 -25
- package/adapters/opencode/opencode.json +1 -1
- package/adapters/workos/WORKOS.md +52 -0
- package/bin/cli.js +373 -5
- package/bin/futureagi-bridge +9 -0
- package/config/gate-templates.json +653 -4
- package/config/gates/actor-critic-audit.json +34 -0
- package/config/gates/default.json +21 -2
- package/config/gates/five-walls-governance.json +34 -0
- package/config/gates/future-agi-guardrails.json +34 -0
- package/config/gates/radware-threat-defense-2026.json +61 -0
- package/config/gates/simatree-data-governance.json +33 -0
- package/config/mcp-allowlists.json +4 -0
- package/config/merge-quality-checks.json +10 -1
- package/config/model-candidates.json +382 -24
- package/config/model-tiers.json +18 -0
- package/config/post-deploy-marketing-pages.json +10 -0
- package/config/progressive/01-wire-only.json +11 -0
- package/config/progressive/02-dashboard-empty-ok.json +10 -0
- package/config/progressive/03-one-lesson.json +10 -0
- package/config/progressive/04-warn-fires.json +11 -0
- package/config/progressive/05-strict-optional.json +11 -0
- package/config/progressive/README.md +15 -0
- package/config/schemas/broker-execution-receipt.schema.json +139 -0
- package/config/schemas/provider-execution-attestation-v1.schema.json +58 -0
- package/conformance/provider-attestation/vectors.json +320 -0
- package/docs/specs/provider-execution-attestation-v1.md +69 -0
- package/openapi/openapi.yaml +15 -0
- package/package.json +401 -147
- package/public/about.html +2 -2
- package/public/ai-malpractice-prevention.html +7 -7
- package/public/blog/a-10-dollar-vps-is-not-a-computer.html +143 -0
- package/public/blog/a-receipt-is-not-world-state.html +388 -0
- package/public/blog/git-at-agent-scale.html +374 -0
- package/public/blog/no-llm-in-the-gate.html +133 -0
- package/public/blog.html +80 -0
- package/public/case-studies.html +16 -1
- package/public/compare.html +28 -0
- package/public/diagnostic.html +216 -7
- package/public/docs/connectors.html +39 -0
- package/public/federal.html +2 -2
- package/public/founders.html +639 -0
- package/public/index.html +87 -9
- package/public/install.html +8 -8
- package/public/learn.html +39 -0
- package/public/numbers.html +2 -2
- package/public/peter.html +310 -0
- package/public/platform-partners.html +119 -0
- package/public/pricing.html +24 -3
- package/public/privacy.html +117 -0
- package/public/pro.html +17 -0
- package/public/support.html +62 -0
- package/public/terms.html +130 -0
- package/public/third-party-notices.html +95 -0
- package/public/yt.html +351 -0
- package/scripts/action-receipts.js +133 -3
- package/scripts/adaptive-governance-arena.js +349 -0
- package/scripts/admin-override.js +205 -0
- package/scripts/agent-action-inventory.js +869 -0
- package/scripts/agent-audit-trace.js +42 -2
- package/scripts/agent-egress-policy.js +1117 -0
- package/scripts/agent-memory-lifecycle.js +141 -2
- package/scripts/agent-operations-planner.js +441 -1
- package/scripts/agent-readiness.js +68 -0
- package/scripts/agent-security-central.js +647 -0
- package/scripts/allowlist-bridge-honesty.js +417 -0
- package/scripts/async-job-runner.js +102 -11
- package/scripts/audit-trail.js +212 -0
- package/scripts/auto-promote-gates.js +178 -27
- package/scripts/billing.js +1 -1
- package/scripts/broker-execution-receipts.js +719 -0
- package/scripts/budget-aware-gates-proof.js +423 -0
- package/scripts/claude-feedback-sync.js +29 -3
- package/scripts/claw-harness-production.js +237 -0
- package/scripts/cli-progress.js +111 -0
- package/scripts/cli-schema.js +163 -1
- package/scripts/codex-runbook-flywheel.js +318 -0
- package/scripts/context-footprint.js +186 -0
- package/scripts/contextfs.js +143 -61
- package/scripts/dashboard-limits.js +27 -0
- package/scripts/dashboard.js +279 -9
- package/scripts/deepseek-v4-runtime-guardrails.js +72 -6
- package/scripts/docker-sandbox-planner.js +18 -0
- package/scripts/double-blind-eval-protocol.js +252 -0
- package/scripts/edotenv-rl-gateway.js +259 -0
- package/scripts/ensure-production-search-corpus.js +162 -0
- package/scripts/eval-holdout.js +311 -0
- package/scripts/feedback-aggregate.js +21 -2
- package/scripts/feedback-loop.js +87 -5
- package/scripts/feedback-quality.js +9 -0
- package/scripts/file-ledger-lock.js +4 -1
- package/scripts/financial-control-plane.js +41 -1
- package/scripts/find-dormant-requires.js +118 -0
- package/scripts/fs-utils.js +84 -8
- package/scripts/gate-stats.js +2 -2
- package/scripts/gates-engine.js +859 -58
- package/scripts/generate-case-study-outreach.js +24 -15
- package/scripts/git-at-scale.js +628 -0
- package/scripts/governance-conflict-audit.js +1650 -0
- package/scripts/governance-difficulty-curriculum.js +328 -0
- package/scripts/graphrag-retrieval.js +275 -0
- package/scripts/gurobi-optimizer.js +324 -0
- package/scripts/gurobi_optimizer.py +485 -0
- package/scripts/harness-selector.js +82 -1
- package/scripts/hidden-entry-points.js +284 -0
- package/scripts/human-escalation.js +199 -1
- package/scripts/hybrid-feedback-context.js +152 -19
- package/scripts/intent-governed-execution.js +602 -0
- package/scripts/intervention-policy.js +123 -20
- package/scripts/jit-harness-compose.js +628 -0
- package/scripts/jsonl-watcher.js +10 -0
- package/scripts/lesson-embedding-index.js +95 -12
- package/scripts/lesson-retrieval.js +105 -19
- package/scripts/local-model-profile.js +19 -2
- package/scripts/mailer/resend-mailer.js +1 -1
- package/scripts/matryoshka-embedding.js +235 -0
- package/scripts/mcp-oauth.js +42 -4
- package/scripts/mcp-session-handles.js +1016 -0
- package/scripts/mcp-wiring-doctor.js +314 -0
- package/scripts/memory-firewall.js +115 -2
- package/scripts/memory-scope-readiness.js +299 -0
- package/scripts/memory-vs-rag-route.js +161 -0
- package/scripts/model-tier-router.js +148 -21
- package/scripts/nvidia-specdecode-al-doctor.js +536 -0
- package/scripts/openui-catalog-compose-honesty.js +593 -0
- package/scripts/operational-integrity.js +19 -1
- package/scripts/override-audit.js +213 -0
- package/scripts/package-manager-honesty-doctor.js +458 -0
- package/scripts/pr-manager.js +63 -1
- package/scripts/prove-herdr-adapter.js +52 -0
- package/scripts/prove-memory-pyramid-and-symbolic-canvas.js +95 -0
- package/scripts/prove-workos.js +73 -0
- package/scripts/provider-attestation-conformance.js +192 -0
- package/scripts/provider-receipt-contract.js +136 -0
- package/scripts/qwen38-max-cost-optimizer.js +401 -0
- package/scripts/radware-threat-defense.js +280 -0
- package/scripts/rag-embedding-identity.js +221 -0
- package/scripts/rag-precision-guardrails.js +112 -2
- package/scripts/remote-feedback-capture.js +159 -0
- package/scripts/research-agent-harness.js +256 -0
- package/scripts/rsi-safety-hillclimb.js +200 -0
- package/scripts/rule-sprawl.js +188 -0
- package/scripts/schedule-manager.js +147 -0
- package/scripts/self-heal.js +8 -0
- package/scripts/session-lease.js +415 -0
- package/scripts/simatree-data-governance.js +347 -0
- package/scripts/slo-alert-engine.js +172 -7
- package/scripts/solver-parity.js +539 -0
- package/scripts/stealth-memory-injection-gate.js +333 -0
- package/scripts/switchyard-router.js +366 -0
- package/scripts/telemetry-analytics.js +84 -27
- package/scripts/temporal-decay-weighting.js +138 -0
- package/scripts/test-all.js +165 -0
- package/scripts/token-savings.js +42 -0
- package/scripts/tool-kpi-tracker.js +108 -5
- package/scripts/tool-registry.js +193 -5
- package/scripts/universal-claim-evaluator.js +14 -2
- package/scripts/vector-store.js +279 -9
- package/scripts/workflow-notebook.js +391 -0
- package/scripts/workflow-sentinel.js +111 -12
- package/scripts/workos-production-guard.js +260 -0
- package/scripts/workspace-search-route.js +515 -0
- package/server.json +2 -2
- package/src/agent-identity-boundary.js +76 -0
- package/src/agent-retrieval-cache.js +155 -0
- package/src/alert-noise-ledger.js +502 -0
- package/src/api/server.js +802 -185
- package/src/git-fast-cache.js +220 -0
- package/src/git-wal-sync.js +156 -0
- package/src/hash-anchored-edit.js +82 -0
- package/src/hermes-platform-protocol.js +475 -0
- package/src/hermes-sync-plane.js +241 -0
- package/src/index.js +30 -1
- package/src/iso42001-compliance-guard.js +97 -0
- package/src/latency-budget.js +244 -0
- package/src/mcp-writeguard.js +316 -0
- package/src/miminions-adapter.js +106 -0
- package/src/pipeline-compass.js +104 -0
- package/src/ppl-alert-pipeline.js +284 -0
- package/src/rendezvous-router.js +90 -0
- package/src/security-questionnaire.js +195 -0
package/scripts/dashboard.js
CHANGED
|
@@ -53,6 +53,11 @@ const {
|
|
|
53
53
|
collectAggregateLogEntries,
|
|
54
54
|
shouldAggregateFeedback,
|
|
55
55
|
} = require('./feedback-aggregate');
|
|
56
|
+
const {
|
|
57
|
+
DEFAULT_JSONL_TAIL_BYTES,
|
|
58
|
+
DEFAULT_JSONL_TAIL_ENTRIES,
|
|
59
|
+
readTextTail,
|
|
60
|
+
} = require('./fs-utils');
|
|
56
61
|
|
|
57
62
|
const PROJECT_ROOT = path.join(__dirname, '..');
|
|
58
63
|
const DEFAULT_GATES_PATH = path.join(PROJECT_ROOT, 'config', 'gates', 'default.json');
|
|
@@ -97,13 +102,41 @@ function buildUnavailableOrgDashboard(windowHours) {
|
|
|
97
102
|
// Data readers
|
|
98
103
|
// ---------------------------------------------------------------------------
|
|
99
104
|
|
|
100
|
-
|
|
105
|
+
// Prod feedback/memory logs can grow past V8's max string size (~512MB–1GB).
|
|
106
|
+
// Full-file readFileSync/stringify then throws:
|
|
107
|
+
// "Cannot create a string longer than 0x1fffffe8 characters"
|
|
108
|
+
// Cap dashboard JSONL ingestion to a recent tail so /v1/dashboard stays live.
|
|
109
|
+
// A 32 MiB default is still too large: generateDashboard reads several logs
|
|
110
|
+
// and aggregation/analyze used to full-scan the same files again.
|
|
111
|
+
const DEFAULT_JSONL_MAX_BYTES = DEFAULT_JSONL_TAIL_BYTES;
|
|
112
|
+
const DEFAULT_JSONL_MAX_ENTRIES = DEFAULT_JSONL_TAIL_ENTRIES;
|
|
113
|
+
|
|
114
|
+
function readJSONL(filePath, options = {}) {
|
|
101
115
|
if (!fs.existsSync(filePath)) return [];
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
116
|
+
const requestedBytes = Number(options.maxBytes);
|
|
117
|
+
const requestedEntries = Number(options.maxEntries);
|
|
118
|
+
const maxBytes = requestedBytes > 0 ? requestedBytes : DEFAULT_JSONL_MAX_BYTES;
|
|
119
|
+
const maxEntries = requestedEntries > 0 ? requestedEntries : DEFAULT_JSONL_MAX_ENTRIES;
|
|
120
|
+
let text;
|
|
121
|
+
try {
|
|
122
|
+
text = readTextTail(filePath, maxBytes).text;
|
|
123
|
+
} catch (err) {
|
|
124
|
+
// Never let a single bloated/unreadable log take down the whole dashboard.
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
if (!text?.trim()) return [];
|
|
128
|
+
const lines = text.split('\n');
|
|
129
|
+
const start = Math.max(0, lines.length - maxEntries);
|
|
130
|
+
const entries = [];
|
|
131
|
+
for (let i = start; i < lines.length; i += 1) {
|
|
132
|
+
const line = lines[i];
|
|
133
|
+
if (!line) continue;
|
|
134
|
+
try {
|
|
135
|
+
const parsed = JSON.parse(line);
|
|
136
|
+
if (parsed) entries.push(parsed);
|
|
137
|
+
} catch { /* skip bad line */ }
|
|
138
|
+
}
|
|
139
|
+
return entries;
|
|
107
140
|
}
|
|
108
141
|
|
|
109
142
|
function readJsonFile(filePath) {
|
|
@@ -523,8 +556,14 @@ function computePreventionImpact(feedbackDir, gateStats) {
|
|
|
523
556
|
}
|
|
524
557
|
|
|
525
558
|
// Estimate time saved: ~16 min per blocked action (conservative)
|
|
559
|
+
// Based on: Claude Code review takes ~30 min, guard saves ~14 min
|
|
526
560
|
const estimatedMinutesSaved = gateStats.blocked * 16;
|
|
527
|
-
const estimatedHoursSaved = (estimatedMinutesSaved / 60).toFixed(1);
|
|
561
|
+
const estimatedHoursSaved = Number((estimatedMinutesSaved / 60).toFixed(1));
|
|
562
|
+
|
|
563
|
+
// Estimate cost savings: $0.004 per blocked action (16 min @ $1.50/hr = 40 cents)
|
|
564
|
+
// This is the cost of the agent time saved
|
|
565
|
+
const estimatedCostSavingsCents = gateStats.blocked * (16 / 60) * 1.5 * 100; // 16min * $1.50/hr -> cents
|
|
566
|
+
const estimatedCostSavings = `$${(estimatedCostSavingsCents / 100).toFixed(2)}`;
|
|
528
567
|
|
|
529
568
|
// Last auto-promotion
|
|
530
569
|
const autoGates = readJsonFile(autoGatesPath);
|
|
@@ -540,10 +579,71 @@ function computePreventionImpact(feedbackDir, gateStats) {
|
|
|
540
579
|
}
|
|
541
580
|
}
|
|
542
581
|
|
|
582
|
+
// Feedback-to-gate conversion rate
|
|
583
|
+
// How many blocked actions led to prevention rules — rate per blocked event
|
|
584
|
+
const feedbackCount = gateStats.blocked; // denominator: blocked actions
|
|
585
|
+
const conversionRate = feedbackCount > 0 ? Math.min(gateStats.totalGates / feedbackCount, 1) : 0;
|
|
586
|
+
|
|
543
587
|
return {
|
|
544
588
|
estimatedHoursSaved,
|
|
589
|
+
estimatedCostSavings,
|
|
545
590
|
ruleCount,
|
|
546
591
|
lastPromotion,
|
|
592
|
+
feedbackToGateConversionRate: Number(conversionRate.toFixed(4)),
|
|
593
|
+
blockedActions: gateStats.blocked || 0,
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function computeRoiSummary(gateStats, prevention, analytics, billing) {
|
|
598
|
+
// North Star: Earn $100/day after-tax profit
|
|
599
|
+
// ROI metrics that drive acquisition and retention
|
|
600
|
+
|
|
601
|
+
const blockedActions = gateStats.blocked || 0;
|
|
602
|
+
const warnedActions = gateStats.warned || 0;
|
|
603
|
+
const passedActions = gateStats.passed || 0;
|
|
604
|
+
|
|
605
|
+
// Time saved from blocked actions (16 min per block)
|
|
606
|
+
const hoursSaved = prevention.estimatedHoursSaved || 0;
|
|
607
|
+
|
|
608
|
+
// Conversion funnel effectiveness
|
|
609
|
+
const funnel = analytics.funnel || {};
|
|
610
|
+
const visitorToPaidRate = funnel.visitorToPaidRate || 0;
|
|
611
|
+
const ctaToPaidRate = funnel.ctaToPaidRate || 0;
|
|
612
|
+
|
|
613
|
+
// Revenue tracking — aligns with billing.js getBillingSummary: revenue.bookedRevenueCents
|
|
614
|
+
const paidOrders = funnel.paidOrders || 0;
|
|
615
|
+
const monthlyRevenueCents = billing.revenue ? billing.revenue.bookedRevenueCents || 0 : 0;
|
|
616
|
+
const monthlyRevenue = monthlyRevenueCents / 100;
|
|
617
|
+
|
|
618
|
+
// Prevention effectiveness: blocked actions per attempted action
|
|
619
|
+
const totalActions = blockedActions + warnedActions + passedActions;
|
|
620
|
+
const preventionRate = totalActions > 0 ? (blockedActions / totalActions) : 0;
|
|
621
|
+
|
|
622
|
+
// PR merge efficiency (GitHub Label Archiving impact)
|
|
623
|
+
const decisionMetrics = analytics.decisionMetrics || {};
|
|
624
|
+
const prMergeRate = decisionMetrics.mergeRate || 0;
|
|
625
|
+
|
|
626
|
+
return {
|
|
627
|
+
// Cost savings from prevention (time-based ROI)
|
|
628
|
+
hoursSaved: Number(hoursSaved.toFixed(1)),
|
|
629
|
+
preventionRate: Number(preventionRate.toFixed(4)),
|
|
630
|
+
blockedActions,
|
|
631
|
+
|
|
632
|
+
// Conversion metrics (acquisition ROI)
|
|
633
|
+
visitorToPaidRate: Number((visitorToPaidRate * 100).toFixed(2)),
|
|
634
|
+
ctaToPaidRate: Number((ctaToPaidRate * 100).toFixed(2)),
|
|
635
|
+
|
|
636
|
+
// Revenue signals (retention/retention)
|
|
637
|
+
monthlyRevenue: Number(monthlyRevenue.toFixed(2)),
|
|
638
|
+
paidOrders,
|
|
639
|
+
|
|
640
|
+
// North Star tracking
|
|
641
|
+
monthlyRevenueTarget: 3000, // $100/day after tax
|
|
642
|
+
revenueProgress: monthlyRevenue > 0 ? (monthlyRevenue / 3000 * 100).toFixed(1) : 0,
|
|
643
|
+
|
|
644
|
+
// Process efficiency (GitHub Label Archiving benefits)
|
|
645
|
+
prMergeRate: Number((prMergeRate * 100).toFixed(1)),
|
|
646
|
+
// Higher merge rate = better process efficiency from archived labels reducing noise
|
|
547
647
|
};
|
|
548
648
|
}
|
|
549
649
|
|
|
@@ -1160,6 +1260,149 @@ function computeObservabilityStats(diagnosticEntries, diagnostics, secretGuard,
|
|
|
1160
1260
|
};
|
|
1161
1261
|
}
|
|
1162
1262
|
|
|
1263
|
+
// ---------------------------------------------------------------------------
|
|
1264
|
+
// Agent File Audit (high-ROI improvement for agent governance)
|
|
1265
|
+
// ---------------------------------------------------------------------------
|
|
1266
|
+
|
|
1267
|
+
function computeAgentFileAudit(projectRoot = PROJECT_ROOT) {
|
|
1268
|
+
// Audit agent configuration files for governance compliance
|
|
1269
|
+
const auditResults = {
|
|
1270
|
+
totalAgentFiles: 0,
|
|
1271
|
+
auditPassCount: 0,
|
|
1272
|
+
auditFailCount: 0,
|
|
1273
|
+
files: [],
|
|
1274
|
+
lastAuditAt: new Date().toISOString(),
|
|
1275
|
+
};
|
|
1276
|
+
|
|
1277
|
+
// Find all agent/MCP configuration files
|
|
1278
|
+
const agentFilePatterns = [
|
|
1279
|
+
'.mcp.json',
|
|
1280
|
+
'.cursor/mcp.json',
|
|
1281
|
+
'.claude/claude.json',
|
|
1282
|
+
'.gemini/rules.json',
|
|
1283
|
+
];
|
|
1284
|
+
|
|
1285
|
+
const agentFiles = [];
|
|
1286
|
+
|
|
1287
|
+
// Scan for agent files
|
|
1288
|
+
for (const pattern of agentFilePatterns) {
|
|
1289
|
+
const fullPath = path.join(projectRoot, pattern);
|
|
1290
|
+
if (fs.existsSync(fullPath)) {
|
|
1291
|
+
agentFiles.push({ path: pattern, fullPath });
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
// Check adapters directory for MCP configs
|
|
1296
|
+
const adaptersDir = path.join(projectRoot, 'adapters');
|
|
1297
|
+
if (fs.existsSync(adaptersDir)) {
|
|
1298
|
+
const entries = fs.readdirSync(adaptersDir, { withFileTypes: true });
|
|
1299
|
+
for (const entry of entries) {
|
|
1300
|
+
if (entry.isDirectory()) {
|
|
1301
|
+
const mcpPath = path.join(adaptersDir, entry.name, '.mcp.json');
|
|
1302
|
+
if (fs.existsSync(mcpPath)) {
|
|
1303
|
+
agentFiles.push({ path: `adapters/${entry.name}/.mcp.json`, fullPath: mcpPath });
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
// Check plugins directory
|
|
1310
|
+
const pluginsDir = path.join(projectRoot, 'plugins');
|
|
1311
|
+
if (fs.existsSync(pluginsDir)) {
|
|
1312
|
+
const entries = fs.readdirSync(pluginsDir, { withFileTypes: true });
|
|
1313
|
+
for (const entry of entries) {
|
|
1314
|
+
if (entry.isDirectory()) {
|
|
1315
|
+
const mcpPath = path.join(pluginsDir, entry.name, '.mcp.json');
|
|
1316
|
+
if (fs.existsSync(mcpPath)) {
|
|
1317
|
+
agentFiles.push({ path: `plugins/${entry.name}/.mcp.json`, fullPath: mcpPath });
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
auditResults.totalAgentFiles = agentFiles.length;
|
|
1324
|
+
|
|
1325
|
+
// Audit each file for basic governance
|
|
1326
|
+
for (const file of agentFiles) {
|
|
1327
|
+
try {
|
|
1328
|
+
const content = fs.readFileSync(file.fullPath, 'utf-8');
|
|
1329
|
+
const config = JSON.parse(content);
|
|
1330
|
+
|
|
1331
|
+
let auditPass = true;
|
|
1332
|
+
const issues = [];
|
|
1333
|
+
|
|
1334
|
+
// Check for required safety fields
|
|
1335
|
+
if (!config.permissions && !config.allowAllPermissions) {
|
|
1336
|
+
// Not all configs require permissions, so this is optional
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
// Check for proper command structure
|
|
1340
|
+
if (config.mcpServers) {
|
|
1341
|
+
for (const [name, server] of Object.entries(config.mcpServers)) {
|
|
1342
|
+
if (!server.command) {
|
|
1343
|
+
auditPass = false;
|
|
1344
|
+
issues.push(`Server ${name} missing command`);
|
|
1345
|
+
}
|
|
1346
|
+
if (server.cwd && !path.isAbsolute(server.cwd)) {
|
|
1347
|
+
auditPass = false;
|
|
1348
|
+
issues.push(`Server ${name} has relative cwd`);
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
auditResults.auditPassCount += auditPass ? 1 : 0;
|
|
1354
|
+
auditResults.auditFailCount += auditPass ? 0 : 1;
|
|
1355
|
+
auditResults.files.push({
|
|
1356
|
+
path: file.path,
|
|
1357
|
+
auditPass,
|
|
1358
|
+
issues: auditPass ? [] : issues,
|
|
1359
|
+
size: fs.statSync(file.fullPath).size,
|
|
1360
|
+
});
|
|
1361
|
+
} catch (err) {
|
|
1362
|
+
auditResults.auditFailCount += 1;
|
|
1363
|
+
auditResults.files.push({
|
|
1364
|
+
path: file.path,
|
|
1365
|
+
auditPass: false,
|
|
1366
|
+
issues: [`Parse error: ${err.message}`],
|
|
1367
|
+
size: 0,
|
|
1368
|
+
});
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
// Check for server-card.json (MCP registry compatibility)
|
|
1373
|
+
const serverCardPath = path.join(projectRoot, '.well-known', 'mcp', 'server-card.json');
|
|
1374
|
+
if (fs.existsSync(serverCardPath)) {
|
|
1375
|
+
auditResults.serverCard = {
|
|
1376
|
+
present: true,
|
|
1377
|
+
path: '.well-known/mcp/server-card.json',
|
|
1378
|
+
};
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
return auditResults;
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
// ---------------------------------------------------------------------------
|
|
1385
|
+
// Agent Surface Inventory for Pro Dashboard
|
|
1386
|
+
// ---------------------------------------------------------------------------
|
|
1387
|
+
|
|
1388
|
+
function computeAgentFileAuditFromFeedback(feedbackDir, options = {}) {
|
|
1389
|
+
const readiness = options.readiness || generateAgentReadinessReport({ projectRoot: PROJECT_ROOT });
|
|
1390
|
+
const auditResults = computeAgentFileAudit();
|
|
1391
|
+
|
|
1392
|
+
return {
|
|
1393
|
+
totalAgentFiles: auditResults.totalAgentFiles,
|
|
1394
|
+
audited: auditResults.auditPassCount === auditResults.totalAgentFiles,
|
|
1395
|
+
highRiskCount: auditResults.auditFailCount,
|
|
1396
|
+
readinessScore: readiness.score || 0,
|
|
1397
|
+
lastAuditAt: auditResults.lastAuditAt,
|
|
1398
|
+
details: auditResults.files.map(f => ({
|
|
1399
|
+
path: f.path,
|
|
1400
|
+
safe: f.auditPass,
|
|
1401
|
+
issues: f.issues,
|
|
1402
|
+
})),
|
|
1403
|
+
};
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1163
1406
|
function computeInstrumentationReadiness(analytics, billing) {
|
|
1164
1407
|
const landingPage = fs.existsSync(LANDING_PAGE_PATH)
|
|
1165
1408
|
? fs.readFileSync(LANDING_PAGE_PATH, 'utf-8')
|
|
@@ -1481,6 +1724,16 @@ function computeAgentSurfaceInventory(feedbackDir, options = {}) {
|
|
|
1481
1724
|
writeCapableTools: readiness.permissions.writeCapableTools.slice(0, 8),
|
|
1482
1725
|
activeBootstrapFiles: readiness.bootstrap.requiredPresent,
|
|
1483
1726
|
requiredBootstrapFiles: readiness.bootstrap.requiredCount,
|
|
1727
|
+
// Merged from computeAgentFileAuditFromFeedback — audit pass/fail for dashboard
|
|
1728
|
+
...(() => {
|
|
1729
|
+
const audit = computeAgentFileAuditFromFeedback(feedbackDir, options);
|
|
1730
|
+
return {
|
|
1731
|
+
totalAgentFiles: audit.totalAgentFiles,
|
|
1732
|
+
audited: audit.audited,
|
|
1733
|
+
highRiskCount: audit.highRiskCount,
|
|
1734
|
+
lastAuditAt: audit.lastAuditAt,
|
|
1735
|
+
};
|
|
1736
|
+
})(),
|
|
1484
1737
|
};
|
|
1485
1738
|
}
|
|
1486
1739
|
|
|
@@ -1550,7 +1803,11 @@ function resolveTeamWindowHours(analyticsWindow) {
|
|
|
1550
1803
|
|
|
1551
1804
|
function collectAllFeedbackEntries(feedbackDir) {
|
|
1552
1805
|
if (shouldAggregateFeedback()) {
|
|
1553
|
-
return collectAggregateLogEntries('feedback-log.jsonl', {
|
|
1806
|
+
return collectAggregateLogEntries('feedback-log.jsonl', {
|
|
1807
|
+
feedbackDir,
|
|
1808
|
+
maxLines: DEFAULT_JSONL_MAX_ENTRIES,
|
|
1809
|
+
maxBytes: DEFAULT_JSONL_MAX_BYTES,
|
|
1810
|
+
}).entries;
|
|
1554
1811
|
}
|
|
1555
1812
|
|
|
1556
1813
|
const entries = [];
|
|
@@ -1652,7 +1909,11 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
1652
1909
|
availability: 'private_core',
|
|
1653
1910
|
};
|
|
1654
1911
|
const readiness = generateAgentReadinessReport({ projectRoot: PROJECT_ROOT });
|
|
1655
|
-
const feedbackAnalysis = analyzeFeedback(path.join(feedbackDir, 'feedback-log.jsonl')
|
|
1912
|
+
const feedbackAnalysis = analyzeFeedback(path.join(feedbackDir, 'feedback-log.jsonl'), {
|
|
1913
|
+
entries,
|
|
1914
|
+
maxLines: DEFAULT_JSONL_MAX_ENTRIES,
|
|
1915
|
+
maxBytes: DEFAULT_JSONL_MAX_BYTES,
|
|
1916
|
+
});
|
|
1656
1917
|
const harness = computeHarnessOverview(feedbackDir, entries);
|
|
1657
1918
|
const interventionPolicy = getInterventionPolicySummary(feedbackDir);
|
|
1658
1919
|
const decisionRecords = readDecisionLog(path.join(feedbackDir, DECISION_LOG_FILENAME));
|
|
@@ -1755,6 +2016,9 @@ function generateDashboard(feedbackDir, options = {}) {
|
|
|
1755
2016
|
});
|
|
1756
2017
|
const reviewDelta = summarizeReviewDelta(entries, memoryEntries, auditEntries, reviewBaseline, reviewSnapshot);
|
|
1757
2018
|
|
|
2019
|
+
// High-ROI metrics for North Star: $100/day after-tax profit
|
|
2020
|
+
const roiSummary = computeRoiSummary(gateStats, prevention, analytics, billingSummary);
|
|
2021
|
+
|
|
1758
2022
|
return {
|
|
1759
2023
|
operational: {
|
|
1760
2024
|
source: options.billingSource || 'local',
|
|
@@ -2080,6 +2344,7 @@ module.exports = {
|
|
|
2080
2344
|
computeDecisionMetrics,
|
|
2081
2345
|
computeGateStats,
|
|
2082
2346
|
computePreventionImpact,
|
|
2347
|
+
computeRoiSummary,
|
|
2083
2348
|
computeSessionTrend,
|
|
2084
2349
|
computeSystemHealth,
|
|
2085
2350
|
computeEfficiencyMetrics,
|
|
@@ -2088,9 +2353,14 @@ module.exports = {
|
|
|
2088
2353
|
computeAnalyticsSummary,
|
|
2089
2354
|
computeSecretGuardStats,
|
|
2090
2355
|
computeObservabilityStats,
|
|
2356
|
+
computeAgentFileAudit,
|
|
2357
|
+
computeAgentSurfaceInventory,
|
|
2091
2358
|
readJSONL,
|
|
2359
|
+
readTextTail,
|
|
2092
2360
|
readJsonFile,
|
|
2093
2361
|
collectAllFeedbackEntries,
|
|
2362
|
+
DEFAULT_JSONL_MAX_BYTES,
|
|
2363
|
+
DEFAULT_JSONL_MAX_ENTRIES,
|
|
2094
2364
|
};
|
|
2095
2365
|
|
|
2096
2366
|
if (require.main === module) {
|
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
const { listGateTemplates } = require('./gate-templates');
|
|
5
|
+
const {
|
|
6
|
+
theoreticalSpeedup,
|
|
7
|
+
optimalDraftLengthForAttention,
|
|
8
|
+
isTileAligned,
|
|
9
|
+
} = require('./nvidia-specdecode-al-doctor');
|
|
5
10
|
|
|
6
11
|
const CATEGORY = 'Sparse Attention Runtime Safety';
|
|
7
12
|
|
|
@@ -33,6 +38,11 @@ function normalizeOptions(options = {}) {
|
|
|
33
38
|
cacheCoherenceEval: normalizeBoolean(options['cache-coherence-eval'] || options['cache-eval']),
|
|
34
39
|
speculativeDecoding: normalizeBoolean(options['speculative-decoding'] || options.speculative || options.mtp || options.eagle),
|
|
35
40
|
acceptLength: toNumber(options['accept-length'] || options['spec-accept-length']),
|
|
41
|
+
draftLength: toNumber(options['draft-length'] || options['spec-draft-length'] || options.d),
|
|
42
|
+
draftDepthRatio: toNumber(options['draft-depth-ratio'] || options.rho),
|
|
43
|
+
claimedSpeedup: toNumber(options['claimed-speedup'] || options.speedup),
|
|
44
|
+
queryHeadsPerKvHead: toNumber(options['query-heads-per-kv'] || options.g || options.G),
|
|
45
|
+
attentionDominated: normalizeBoolean(options['attention-dominated'] || options.attention),
|
|
36
46
|
kvOffload,
|
|
37
47
|
training,
|
|
38
48
|
rolloutReplay: normalizeBoolean(options['rollout-replay'] || options.r3),
|
|
@@ -64,7 +74,33 @@ function templateApplicability(template, options) {
|
|
|
64
74
|
return (options.hybridAttention || isLongContext(options)) && (!options.prefixCache || !options.cacheCoherenceEval);
|
|
65
75
|
}
|
|
66
76
|
if (template.id === 'checkpoint-speculative-decoding-acceptance') {
|
|
67
|
-
|
|
77
|
+
if (!options.speculativeDecoding) return false;
|
|
78
|
+
if (options.acceptLength === null || options.acceptLength < 2 || !options.cacheCoherenceEval) return true;
|
|
79
|
+
if (options.draftLength !== null && options.acceptLength > (1 + options.draftLength)) return true;
|
|
80
|
+
const rho = options.draftDepthRatio === null ? 0 : options.draftDepthRatio;
|
|
81
|
+
if (
|
|
82
|
+
options.claimedSpeedup !== null
|
|
83
|
+
&& options.acceptLength !== null
|
|
84
|
+
&& options.draftLength !== null
|
|
85
|
+
) {
|
|
86
|
+
const theory = theoreticalSpeedup(options.acceptLength, options.draftLength, rho);
|
|
87
|
+
if (theory !== null && options.claimedSpeedup > theory) return true;
|
|
88
|
+
}
|
|
89
|
+
if (
|
|
90
|
+
options.attentionDominated
|
|
91
|
+
&& options.queryHeadsPerKvHead !== null
|
|
92
|
+
&& options.draftLength !== null
|
|
93
|
+
) {
|
|
94
|
+
const optimal = optimalDraftLengthForAttention(options.queryHeadsPerKvHead);
|
|
95
|
+
if (
|
|
96
|
+
optimal !== null
|
|
97
|
+
&& options.draftLength > optimal
|
|
98
|
+
&& !isTileAligned(options.queryHeadsPerKvHead, options.draftLength)
|
|
99
|
+
) {
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return false;
|
|
68
104
|
}
|
|
69
105
|
if (template.id === 'require-long-context-kv-offload-capacity-plan') {
|
|
70
106
|
return isLongContext(options) && !options.kvOffload;
|
|
@@ -108,15 +144,27 @@ function hybridAttentionSignal(options) {
|
|
|
108
144
|
}
|
|
109
145
|
|
|
110
146
|
function speculativeDecodingSignal(options) {
|
|
111
|
-
if (!(options.speculativeDecoding || options.acceptLength !== null))
|
|
147
|
+
if (!(options.speculativeDecoding || options.acceptLength !== null || options.draftLength !== null)) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const rho = options.draftDepthRatio === null ? 0 : options.draftDepthRatio;
|
|
151
|
+
const theory = (
|
|
152
|
+
options.acceptLength !== null && options.draftLength !== null
|
|
153
|
+
)
|
|
154
|
+
? theoreticalSpeedup(options.acceptLength, options.draftLength, rho)
|
|
155
|
+
: null;
|
|
112
156
|
return {
|
|
113
157
|
id: 'speculative_decoding',
|
|
114
158
|
label: 'Speculative decoding rollout',
|
|
115
159
|
values: [
|
|
116
160
|
options.speculativeDecoding ? 'speculative decoding enabled' : 'speculative decoding not declared',
|
|
117
|
-
options.acceptLength !== null ? `${options.acceptLength} accept length` : 'accept length missing',
|
|
118
|
-
|
|
119
|
-
|
|
161
|
+
options.acceptLength !== null ? `${options.acceptLength} accept length (AL)` : 'accept length missing',
|
|
162
|
+
options.draftLength !== null ? `${options.draftLength} draft length (D)` : null,
|
|
163
|
+
options.draftDepthRatio !== null ? `ρ=${options.draftDepthRatio}` : null,
|
|
164
|
+
theory !== null ? `theory ${theory}x = AL/(1+ρD)` : null,
|
|
165
|
+
options.claimedSpeedup !== null ? `claimed ${options.claimedSpeedup}x` : null,
|
|
166
|
+
].filter(Boolean),
|
|
167
|
+
risk: 'Draft-token metadata and rollback paths can make throughput claims look good while correctness or acceptance collapses. Reject claims above AL/(1+ρD).',
|
|
120
168
|
};
|
|
121
169
|
}
|
|
122
170
|
|
|
@@ -186,6 +234,18 @@ function buildDeepSeekV4RuntimeGuardrailsPlan(rawOptions = {}, templatesPath) {
|
|
|
186
234
|
newThroughput: options.newThroughput,
|
|
187
235
|
throughputDropPercent: throughputDropPercent(options),
|
|
188
236
|
acceptLength: options.acceptLength,
|
|
237
|
+
draftLength: options.draftLength,
|
|
238
|
+
draftDepthRatio: options.draftDepthRatio,
|
|
239
|
+
claimedSpeedup: options.claimedSpeedup,
|
|
240
|
+
theoreticalSpeedup: (
|
|
241
|
+
options.acceptLength !== null && options.draftLength !== null
|
|
242
|
+
)
|
|
243
|
+
? theoreticalSpeedup(
|
|
244
|
+
options.acceptLength,
|
|
245
|
+
options.draftLength,
|
|
246
|
+
options.draftDepthRatio === null ? 0 : options.draftDepthRatio
|
|
247
|
+
)
|
|
248
|
+
: null,
|
|
189
249
|
trainInferenceDrift: options.trainInferenceDrift,
|
|
190
250
|
},
|
|
191
251
|
summary: {
|
|
@@ -198,11 +258,12 @@ function buildDeepSeekV4RuntimeGuardrailsPlan(rawOptions = {}, templatesPath) {
|
|
|
198
258
|
nextActions: [
|
|
199
259
|
'Benchmark DeepSeek-V4 behind the same ThumbGate eval harness before changing routing defaults.',
|
|
200
260
|
'Require cache-coherence and rollback evidence before enabling hybrid prefix caching or speculative decoding.',
|
|
261
|
+
'Measure AL and D; reject speedup claims above AL/(1+ρD) via nvidia-specdecode-al-doctor.',
|
|
201
262
|
'Keep long-context memory and throughput budgets explicit before raising context windows.',
|
|
202
263
|
'For RL or fine-tuning, require rollout-routing replay, indexer replay, and train-inference drift checks.',
|
|
203
264
|
'Treat FP4/FP8 or mixed-precision paths as gated rollouts until deterministic and sensitive-FP32 checks pass.',
|
|
204
265
|
],
|
|
205
|
-
exampleCommand: 'npx thumbgate deepseek-v4-runtime-guardrails --context-tokens=900000 --hybrid-attention --speculative-decoding --accept-length=1.4 --precision-mode=fp8 --training --json',
|
|
266
|
+
exampleCommand: 'npx thumbgate deepseek-v4-runtime-guardrails --context-tokens=900000 --hybrid-attention --speculative-decoding --accept-length=1.4 --draft-length=7 --draft-depth-ratio=0.05 --claimed-speedup=3 --precision-mode=fp8 --training --json',
|
|
206
267
|
};
|
|
207
268
|
}
|
|
208
269
|
|
|
@@ -221,6 +282,11 @@ function formatDeepSeekV4RuntimeGuardrailsPlan(report) {
|
|
|
221
282
|
if (report.metrics.contextTokens !== null) lines.push(`Context tokens: ${report.metrics.contextTokens}`);
|
|
222
283
|
if (report.metrics.throughputDropPercent !== null) lines.push(`Throughput drop: ${report.metrics.throughputDropPercent}%`);
|
|
223
284
|
if (report.metrics.acceptLength !== null) lines.push(`Spec accept length: ${report.metrics.acceptLength}`);
|
|
285
|
+
if (report.metrics.draftLength !== null) lines.push(`Spec draft length: ${report.metrics.draftLength}`);
|
|
286
|
+
if (report.metrics.theoreticalSpeedup !== null) {
|
|
287
|
+
lines.push(`Spec theory speedup: ${report.metrics.theoreticalSpeedup}x (AL/(1+ρD))`);
|
|
288
|
+
}
|
|
289
|
+
if (report.metrics.claimedSpeedup !== null) lines.push(`Spec claimed speedup: ${report.metrics.claimedSpeedup}x`);
|
|
224
290
|
if (report.metrics.trainInferenceDrift !== null) lines.push(`Train/inference drift: ${report.metrics.trainInferenceDrift}`);
|
|
225
291
|
|
|
226
292
|
if (report.signals.length > 0) {
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
const path = require('node:path');
|
|
5
5
|
|
|
6
6
|
const { classifyCommand } = require('./operational-integrity');
|
|
7
|
+
const { classifyHostRole } = require('./agent-egress-policy');
|
|
7
8
|
|
|
8
9
|
const HIGH_RISK_ACTION_TYPES = new Set([
|
|
9
10
|
'shell.exec',
|
|
@@ -41,15 +42,30 @@ function quoteShellArg(value) {
|
|
|
41
42
|
|
|
42
43
|
function buildNetworkPolicy(input = {}) {
|
|
43
44
|
const allowedHosts = normalizeStringArray(input.allowedHosts || input.egressAllowlist);
|
|
45
|
+
const hosts = allowedHosts.map((host) => classifyHostRole(host));
|
|
46
|
+
const treatAsTrust = input.treatAllowlistAsTrustBoundary === true || input.trusted === true;
|
|
47
|
+
const bridgeHosts = hosts.filter((row) => row.role === 'bridge');
|
|
44
48
|
if (input.requiresNetwork !== true) {
|
|
45
49
|
return {
|
|
46
50
|
mode: 'deny_all',
|
|
47
51
|
allowedHosts: [],
|
|
52
|
+
hosts: [],
|
|
53
|
+
allowlistIsTrustBoundary: false,
|
|
48
54
|
};
|
|
49
55
|
}
|
|
50
56
|
return {
|
|
51
57
|
mode: allowedHosts.length > 0 ? 'allow_list' : 'egress_enabled',
|
|
52
58
|
allowedHosts,
|
|
59
|
+
hosts,
|
|
60
|
+
bridgeHosts: bridgeHosts.map((row) => row.host),
|
|
61
|
+
allowlistIsTrustBoundary: false,
|
|
62
|
+
independentAuthRequired: bridgeHosts.length > 0,
|
|
63
|
+
findings: treatAsTrust && bridgeHosts.length
|
|
64
|
+
? [{
|
|
65
|
+
id: 'allowlist_treated_as_trust_boundary',
|
|
66
|
+
message: 'Docker sandbox allowlist includes package-registry/proxy hops; those are not a trust boundary.',
|
|
67
|
+
}]
|
|
68
|
+
: [],
|
|
53
69
|
};
|
|
54
70
|
}
|
|
55
71
|
|
|
@@ -155,6 +171,8 @@ function buildDockerSandboxPlan(input = {}) {
|
|
|
155
171
|
requiresNetwork: input.requiresNetwork === true || governedCommand || commandInfo.isPublish || actionType === 'upload' || actionType === 'message.send',
|
|
156
172
|
allowedHosts: input.allowedHosts,
|
|
157
173
|
egressAllowlist: input.egressAllowlist,
|
|
174
|
+
treatAllowlistAsTrustBoundary: input.treatAllowlistAsTrustBoundary,
|
|
175
|
+
trusted: input.trusted,
|
|
158
176
|
});
|
|
159
177
|
const launchers = buildLaunchers(workspacePath);
|
|
160
178
|
const summary = buildSummary(shouldSandbox, recommendation);
|