thumbgate 1.29.2 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.well-known/mcp/server-card.json +1 -1
- package/README.md +54 -16
- package/adapters/claude/.mcp.json +2 -2
- package/adapters/forge/forge.yaml +3 -3
- package/adapters/mcp/server-stdio.js +143 -14
- package/adapters/opencode/opencode.json +1 -1
- package/bench/observability-eval-suite.json +2 -2
- package/bin/cli.js +154 -36
- package/config/evals/generation-quality-golden.json +95 -0
- package/config/evals/rag-answer-quality-golden.json +91 -0
- package/config/evals/retrieval-hybrid-ablation.json +66 -0
- package/config/evals/retrieval-ranking-golden.json +522 -0
- package/config/gates/default.json +217 -50
- package/config/mcp-allowlists.json +215 -185
- package/config/model-tiers.json +7 -2
- package/config/post-deploy-marketing-pages.json +26 -1
- package/glama.json +6 -0
- package/package.json +94 -11
- package/public/architecture.html +130 -0
- package/public/assets/diagrams/agent-integration.png +0 -0
- package/public/assets/diagrams/before-after.svg +22 -0
- package/public/assets/diagrams/decision.svg +36 -0
- package/public/assets/diagrams/feedback-pipeline.png +0 -0
- package/public/assets/diagrams/hero-thumbs.svg +68 -0
- package/public/assets/diagrams/loop.svg +40 -0
- package/public/assets/diagrams/plugin-topology.png +0 -0
- package/public/assets/diagrams/pre-action-gate-loop.svg +59 -0
- package/public/assets/diagrams/self-improving-thumbs-loop.svg +105 -0
- package/public/assets/diagrams/stack.svg +18 -0
- package/public/assets/diagrams/thumbgate-architecture.png +0 -0
- package/public/case-studies.html +151 -0
- package/public/compare.html +1 -0
- package/public/dashboard.html +126 -28
- package/public/eval-scorecard.html +195 -0
- package/public/eval-scorecard.json +18 -0
- package/public/evaluations.html +168 -0
- package/public/index.html +143 -13
- package/public/numbers.html +3 -2
- package/public/pricing.html +143 -30
- package/public/whitepaper.html +189 -0
- package/scripts/a-plus-evidence-scorecard.js +303 -0
- package/scripts/activation-quickstart.js +1 -0
- package/scripts/agent-outcome-monitor.js +71 -1
- package/scripts/async-eval-observability.js +36 -11
- package/scripts/audit-trail.js +37 -1
- package/scripts/auto-promote-gates.js +149 -34
- package/scripts/billing.js +3 -1
- package/scripts/claude-feedback-sync.js +3 -2
- package/scripts/cli-feedback.js +13 -7
- package/scripts/colbert-style-maxsim.js +236 -0
- package/scripts/cross-encoder-reranker.js +359 -126
- package/scripts/dashboard-chat.js +350 -17
- package/scripts/document-intake.js +283 -7
- package/scripts/eval-quality-suite.js +204 -0
- package/scripts/feedback-aggregate.js +5 -2
- package/scripts/feedback-loop.js +359 -189
- package/scripts/feedback-paths.js +32 -13
- package/scripts/feedback-quality.js +53 -0
- package/scripts/filesystem-search.js +17 -7
- package/scripts/gates-engine.js +98 -4
- package/scripts/generate-case-study-outreach.js +253 -0
- package/scripts/generate-eval-scorecard.js +276 -0
- package/scripts/growth-campaigns.js +183 -0
- package/scripts/harness-tool-names.js +70 -0
- package/scripts/hook-runtime.js +10 -3
- package/scripts/jsonl-watcher.js +1 -0
- package/scripts/lesson-db.js +16 -5
- package/scripts/lesson-embedding-index.js +67 -20
- package/scripts/lesson-embedding-maintenance.js +177 -0
- package/scripts/lesson-inference.js +23 -4
- package/scripts/lesson-reranker.js +55 -9
- package/scripts/lesson-retrieval.js +375 -32
- package/scripts/lesson-search.js +48 -11
- package/scripts/llm-client.js +304 -15
- package/scripts/mcp-config.js +26 -5
- package/scripts/mcp-oauth.js +37 -2
- package/scripts/model-eval.js +308 -0
- package/scripts/model-tier-router.js +593 -0
- package/scripts/parallel-workflow-orchestrator.js +86 -22
- package/scripts/pragmatic-hybrid-search.js +379 -0
- package/scripts/published-cli.js +11 -1
- package/scripts/rag-document-pipeline.js +461 -0
- package/scripts/rag-structured-output.js +441 -0
- package/scripts/ragas-style-metrics.js +351 -0
- package/scripts/refresh-proof-pack.js +261 -0
- package/scripts/request-envelope.js +178 -0
- package/scripts/rerank-pipeline.js +370 -0
- package/scripts/rerank-quality-eval.js +155 -0
- package/scripts/retrieval-hybrid-ablation.js +120 -0
- package/scripts/retrieval-quality-tier.js +118 -0
- package/scripts/risk-scorer.js +144 -15
- package/scripts/secret-scanner.js +395 -4
- package/scripts/self-distill-agent.js +7 -1
- package/scripts/self-healing-check.js +25 -0
- package/scripts/skill-packs.js +183 -0
- package/scripts/slow-loop.js +72 -0
- package/scripts/statusline-links.js +1 -1
- package/scripts/statusline-local-stats.js +1 -1
- package/scripts/statusline.sh +8 -1
- package/scripts/telemetry-analytics.js +13 -1
- package/scripts/thumbgate-bench.js +13 -0
- package/scripts/thumbgate-search.js +98 -6
- package/scripts/tier-budget-guard.js +186 -0
- package/scripts/tool-kpi-tracker.js +124 -0
- package/scripts/tool-registry.js +95 -1
- package/scripts/vector-store.js +108 -4
- package/scripts/verify-marketing-pages-deployed.js +85 -3
- package/server.json +44 -0
- package/smithery.yaml +17 -0
- package/src/api/server.js +424 -99
|
@@ -32,10 +32,20 @@ const SECRET_FILE_PATTERNS = [
|
|
|
32
32
|
{ id: 'netrc_file', label: 'netrc credentials file', regex: /(^|\/)\.netrc$/i },
|
|
33
33
|
{ id: 'npmrc_file', label: 'npm credentials file', regex: /(^|\/)\.npmrc$/i },
|
|
34
34
|
{ id: 'pypirc_file', label: 'Python package credentials file', regex: /(^|\/)\.pypirc$/i },
|
|
35
|
-
|
|
35
|
+
// Do not match *.pub — public keys are routinely read for SSH setup.
|
|
36
|
+
{ id: 'ssh_private_key', label: 'SSH private key', regex: /(^|\/)(?:id_rsa|id_ed25519|id_dsa|id_ecdsa)$/i },
|
|
37
|
+
{ id: 'ssh_private_key_path', label: 'SSH private key path', regex: /(^|\/)\.ssh\/id_[A-Za-z0-9_-]+$/i },
|
|
38
|
+
{ id: 'aws_credentials', label: 'AWS credentials file', regex: /(^|\/)\.aws\/credentials$/i },
|
|
39
|
+
{ id: 'kubeconfig', label: 'Kubernetes config', regex: /(^|\/)\.kube\/config$/i },
|
|
40
|
+
{ id: 'docker_config', label: 'Docker config credentials', regex: /(^|\/)\.docker\/config\.json$/i },
|
|
41
|
+
{ id: 'gcloud_adc', label: 'GCP application default credentials', regex: /application_default_credentials\.json$/i },
|
|
36
42
|
{ id: 'pem_key_file', label: 'PEM key file', regex: /\.pem$/i },
|
|
37
43
|
];
|
|
38
44
|
|
|
45
|
+
// Env vars that commonly hold secrets. Used to catch exfil when the secret
|
|
46
|
+
// never appears as a literal in the command (e.g. echo $API_KEY | curl ...).
|
|
47
|
+
const SECRET_ENV_VAR_PATTERN = /\$\{?(?:API[_-]?KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GEMINI_API_KEY|AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID|AWS_SESSION_TOKEN|GITHUB_TOKEN|GH_TOKEN|STRIPE_SECRET_KEY|STRIPE_API_KEY|NPM_TOKEN|HF_TOKEN|HUGGINGFACE_TOKEN|DATABASE_URL|DB_PASSWORD|POSTGRES_PASSWORD|MYSQL_PWD|PRIVATE_KEY|SECRET_KEY|ACCESS_TOKEN|AUTH_TOKEN|CLIENT_SECRET|SLACK_BOT_TOKEN|DISCORD_TOKEN)(?!\w)/i;
|
|
48
|
+
|
|
39
49
|
const BASH_SECRET_READ_PREFIXES = [
|
|
40
50
|
'cat',
|
|
41
51
|
'less',
|
|
@@ -52,9 +62,24 @@ const BASH_SECRET_READ_PREFIXES = [
|
|
|
52
62
|
'strings',
|
|
53
63
|
'env',
|
|
54
64
|
'printenv',
|
|
65
|
+
'base64',
|
|
66
|
+
'xxd',
|
|
67
|
+
'od',
|
|
68
|
+
'hexdump',
|
|
55
69
|
];
|
|
56
70
|
|
|
57
|
-
|
|
71
|
+
// Tools that can carry secret material off-box.
|
|
72
|
+
const OUTBOUND_FILE_COMMANDS = new Set(['curl', 'wget', 'http', 'https', 'httpie']);
|
|
73
|
+
const NETWORK_EXFIL_COMMANDS = new Set([
|
|
74
|
+
'curl', 'wget', 'nc', 'ncat', 'netcat', 'scp', 'rsync', 'sftp', 'ftp', 'lftp',
|
|
75
|
+
'http', 'https', 'httpie', 'rclone', 'aws', 'gsutil', 'az', 'gcloud',
|
|
76
|
+
]);
|
|
77
|
+
const SECRET_FILE_READ_COMMANDS = new Set([
|
|
78
|
+
'cat', 'head', 'tail', 'less', 'more', 'base64', 'xxd', 'od', 'hexdump',
|
|
79
|
+
'gzip', 'bzip2', 'xz', 'dd', 'pv', 'tee', 'type', 'bat',
|
|
80
|
+
]);
|
|
81
|
+
// Cloud / object-store verbs that upload local files off-box.
|
|
82
|
+
const CLOUD_UPLOAD_VERBS = new Set(['cp', 'mv', 'sync', 'copy', 'copyto', 'move', 'upload']);
|
|
58
83
|
const OUTBOUND_COMMAND_WRAPPERS = new Set(['command', 'env', 'nohup', 'sudo']);
|
|
59
84
|
const SHELL_QUOTES = new Set(['"', "'"]);
|
|
60
85
|
const SHELL_SEGMENT_SEPARATORS = new Set([';', '|', '&', '\n']);
|
|
@@ -66,6 +91,8 @@ const WRAPPER_OPTIONS_WITH_VALUE = {
|
|
|
66
91
|
'-D', '--chdir',
|
|
67
92
|
]),
|
|
68
93
|
};
|
|
94
|
+
// Note: --data-raw does NOT treat @ as a file reference (curl man page).
|
|
95
|
+
// Keep it out so literal @payload strings are not false-positive scanned.
|
|
69
96
|
const CURL_DATA_FILE_OPTIONS = new Set([
|
|
70
97
|
'-d',
|
|
71
98
|
'--data',
|
|
@@ -228,6 +255,8 @@ function heuristicScanText(text, source = 'text') {
|
|
|
228
255
|
function classifySecretPath(filePath) {
|
|
229
256
|
const normalized = String(filePath || '').trim();
|
|
230
257
|
if (!normalized) return null;
|
|
258
|
+
// Public keys are not secret material (id_ed25519.pub, id_rsa.pub, …).
|
|
259
|
+
if (/\.pub$/i.test(normalized)) return null;
|
|
231
260
|
for (const pattern of SECRET_FILE_PATTERNS) {
|
|
232
261
|
if (pattern.regex.test(normalized)) {
|
|
233
262
|
return {
|
|
@@ -610,21 +639,377 @@ function scanOutboundCommandFiles(command, cwd, provider) {
|
|
|
610
639
|
const findings = [];
|
|
611
640
|
const fileHashes = [];
|
|
612
641
|
for (const reference of extractOutboundFileReferences(command, cwd)) {
|
|
642
|
+
// Path findings count: curl --data-binary @.env must deny even when the
|
|
643
|
+
// file is missing or empty. Content findings still apply when present.
|
|
613
644
|
const fileScan = scanFile(reference.path, {
|
|
614
645
|
provider,
|
|
615
|
-
includePathFinding:
|
|
646
|
+
includePathFinding: true,
|
|
616
647
|
});
|
|
617
648
|
if (!fileScan.detected) continue;
|
|
618
649
|
findings.push(...fileScan.findings.map((finding) => ({
|
|
619
650
|
...finding,
|
|
620
651
|
source: 'outbound_file',
|
|
621
|
-
reason:
|
|
652
|
+
reason: finding.reason
|
|
653
|
+
|| `${finding.label} found in file referenced by ${reference.command} ${reference.option}`,
|
|
622
654
|
})));
|
|
623
655
|
if (fileScan.fileHash) fileHashes.push(fileScan.fileHash);
|
|
624
656
|
}
|
|
625
657
|
return { findings, fileHashes };
|
|
626
658
|
}
|
|
627
659
|
|
|
660
|
+
function classifySecretPathToken(token) {
|
|
661
|
+
const raw = String(token || '').trim().replace(/^["']|["']$/g, '');
|
|
662
|
+
if (!raw || raw === '-') return null;
|
|
663
|
+
// Strip curl @file / form field=@file / <file prefixes.
|
|
664
|
+
let candidate = raw;
|
|
665
|
+
if (candidate.startsWith('@')) candidate = candidate.slice(1);
|
|
666
|
+
const formAt = candidate.indexOf('=@');
|
|
667
|
+
if (formAt > 0) candidate = candidate.slice(formAt + 2);
|
|
668
|
+
const formLt = candidate.indexOf('=<');
|
|
669
|
+
if (formLt > 0) candidate = candidate.slice(formLt + 2);
|
|
670
|
+
candidate = candidate.split(';', 1)[0];
|
|
671
|
+
// Drop remote scp host:path → keep local path side when present.
|
|
672
|
+
if (/^[A-Za-z0-9._-]+@[^:]+:/.test(candidate)) return null;
|
|
673
|
+
return classifySecretPath(candidate) || classifySecretPath(path.basename(candidate));
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function segmentHasNetworkExfil(segment) {
|
|
677
|
+
const tokens = tokenizeCommand(segment);
|
|
678
|
+
for (const token of tokens) {
|
|
679
|
+
const verb = path.basename(String(token || '')).toLowerCase();
|
|
680
|
+
if (NETWORK_EXFIL_COMMANDS.has(verb)) return verb;
|
|
681
|
+
}
|
|
682
|
+
return null;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
function segmentSecretPathFindings(segment, cwd) {
|
|
686
|
+
const findings = [];
|
|
687
|
+
const tokens = tokenizeCommand(segment);
|
|
688
|
+
for (const token of tokens) {
|
|
689
|
+
const pathFinding = classifySecretPathToken(token);
|
|
690
|
+
if (!pathFinding) continue;
|
|
691
|
+
const resolved = resolvePathToken(
|
|
692
|
+
String(token).replace(/^@/, '').split('=@').pop().split('=<').pop().split(';', 1)[0],
|
|
693
|
+
cwd
|
|
694
|
+
);
|
|
695
|
+
findings.push({
|
|
696
|
+
...pathFinding,
|
|
697
|
+
path: resolved || pathFinding.path,
|
|
698
|
+
source: 'exfil_path',
|
|
699
|
+
reason: `${pathFinding.label} referenced in potential exfiltration command`,
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
return findings;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function detectCommandSubstitutionExfil(command, cwd) {
|
|
706
|
+
// $(cat .env), $(base64 .env), `cat .env`, <(cat .env) — secret not literal in curl argv.
|
|
707
|
+
const findings = [];
|
|
708
|
+
const patterns = [
|
|
709
|
+
/\$\(\s*([a-z0-9._+-]+)\s+([^)]+)\)/gi,
|
|
710
|
+
/`\s*([a-z0-9._+-]+)\s+([^`]+)`/gi,
|
|
711
|
+
/<\(\s*([a-z0-9._+-]+)\s+([^)]+)\)/gi,
|
|
712
|
+
];
|
|
713
|
+
for (const pattern of patterns) {
|
|
714
|
+
pattern.lastIndex = 0;
|
|
715
|
+
let match = pattern.exec(command);
|
|
716
|
+
while (match) {
|
|
717
|
+
const verb = path.basename(String(match[1] || '')).toLowerCase();
|
|
718
|
+
if (!SECRET_FILE_READ_COMMANDS.has(verb)) {
|
|
719
|
+
match = pattern.exec(command);
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
722
|
+
const args = tokenizeCommand(match[2] || '');
|
|
723
|
+
for (const arg of args) {
|
|
724
|
+
const pathFinding = classifySecretPathToken(arg);
|
|
725
|
+
if (!pathFinding) continue;
|
|
726
|
+
findings.push({
|
|
727
|
+
...pathFinding,
|
|
728
|
+
source: 'command_substitution',
|
|
729
|
+
reason: `${pathFinding.label} read via command substitution (${verb}) for possible exfiltration`,
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
match = pattern.exec(command);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
// Only treat as exfil when a real network/transfer sink is present.
|
|
736
|
+
// Do not treat a URL mentioned in documentation text as a sink.
|
|
737
|
+
if (findings.length === 0) return [];
|
|
738
|
+
const hasNetwork = Boolean(segmentHasNetworkExfil(command))
|
|
739
|
+
|| /\/dev\/tcp\//i.test(command);
|
|
740
|
+
if (!hasNetwork) return [];
|
|
741
|
+
return findings;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
function detectPipelineExfil(command, cwd) {
|
|
745
|
+
// cat .env | nc … ; base64 .env | curl … ; echo $API_KEY | curl …
|
|
746
|
+
const findings = [];
|
|
747
|
+
const segments = String(command || '').split(/(?<![|])\|(?![|])/);
|
|
748
|
+
if (segments.length < 2) return findings;
|
|
749
|
+
|
|
750
|
+
for (let i = 0; i < segments.length - 1; i += 1) {
|
|
751
|
+
const left = segments[i];
|
|
752
|
+
const right = segments.slice(i + 1).join('|');
|
|
753
|
+
const networkVerb = segmentHasNetworkExfil(right);
|
|
754
|
+
if (!networkVerb) continue;
|
|
755
|
+
|
|
756
|
+
const pathFindings = segmentSecretPathFindings(left, cwd);
|
|
757
|
+
for (const finding of pathFindings) {
|
|
758
|
+
findings.push({
|
|
759
|
+
...finding,
|
|
760
|
+
source: 'pipeline_exfil',
|
|
761
|
+
reason: `${finding.label} piped into ${networkVerb}`,
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
SECRET_ENV_VAR_PATTERN.lastIndex = 0;
|
|
766
|
+
if (SECRET_ENV_VAR_PATTERN.test(left) || SECRET_ENV_VAR_PATTERN.test(right)) {
|
|
767
|
+
findings.push({
|
|
768
|
+
id: 'secret_env_exfil',
|
|
769
|
+
label: 'Secret environment variable',
|
|
770
|
+
source: 'pipeline_exfil',
|
|
771
|
+
reason: `Secret-bearing environment variable piped into ${networkVerb}`,
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
return findings;
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function detectScpRsyncExfil(command, cwd) {
|
|
779
|
+
const findings = [];
|
|
780
|
+
for (const segment of splitCommandSegments(command)) {
|
|
781
|
+
const tokens = tokenizeCommand(segment);
|
|
782
|
+
if (!tokens.length) continue;
|
|
783
|
+
let index = 0;
|
|
784
|
+
while (isShellAssignment(tokens[index])) index += 1;
|
|
785
|
+
const verb = path.basename(String(tokens[index] || '')).toLowerCase();
|
|
786
|
+
if (verb !== 'scp' && verb !== 'rsync' && verb !== 'sftp') continue;
|
|
787
|
+
for (const token of tokens.slice(index + 1)) {
|
|
788
|
+
if (String(token).startsWith('-')) continue;
|
|
789
|
+
// scp local secret → remote OR remote → local (still sensitive move)
|
|
790
|
+
const localSide = String(token).includes(':') && !String(token).startsWith('/')
|
|
791
|
+
? String(token).split(':').slice(1).join(':')
|
|
792
|
+
: token;
|
|
793
|
+
const pathFinding = classifySecretPathToken(localSide) || classifySecretPathToken(token);
|
|
794
|
+
if (!pathFinding) continue;
|
|
795
|
+
findings.push({
|
|
796
|
+
...pathFinding,
|
|
797
|
+
source: 'transfer_exfil',
|
|
798
|
+
reason: `${pathFinding.label} referenced by ${verb} transfer`,
|
|
799
|
+
});
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
return findings;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function detectEnvVarNetworkExfil(command) {
|
|
806
|
+
// echo $API_KEY | curl OR curl -d "$OPENAI_API_KEY" https://…
|
|
807
|
+
// printenv API_KEY | curl
|
|
808
|
+
// Require a parsed network/transfer verb or /dev/tcp — a URL in docs text alone
|
|
809
|
+
// must not hard-deny (e.g. printf 'Set $OPENAI_API_KEY then visit https://…').
|
|
810
|
+
const text = String(command || '');
|
|
811
|
+
SECRET_ENV_VAR_PATTERN.lastIndex = 0;
|
|
812
|
+
const hasEnvRef = SECRET_ENV_VAR_PATTERN.test(text)
|
|
813
|
+
|| /\bprintenv\s+(?:API[_-]?KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|GITHUB_TOKEN|GH_TOKEN|STRIPE_SECRET_KEY|NPM_TOKEN|ACCESS_TOKEN|CLIENT_SECRET)\b/i.test(text)
|
|
814
|
+
|| /\benv\s+(?:API[_-]?KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|GITHUB_TOKEN)\b/i.test(text);
|
|
815
|
+
if (!hasEnvRef) return [];
|
|
816
|
+
const networkVerb = segmentHasNetworkExfil(command);
|
|
817
|
+
if (!networkVerb && !/\/dev\/tcp\//i.test(command)) {
|
|
818
|
+
return [];
|
|
819
|
+
}
|
|
820
|
+
return [{
|
|
821
|
+
id: 'secret_env_exfil',
|
|
822
|
+
label: 'Secret environment variable',
|
|
823
|
+
source: 'env_exfil',
|
|
824
|
+
reason: `Secret-bearing environment variable used with network tool (${networkVerb || '/dev/tcp'})`,
|
|
825
|
+
}];
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function detectRedirectExfil(command, cwd) {
|
|
829
|
+
// curl … < .env | nc host port < ~/.aws/credentials | … > /dev/tcp/host/port
|
|
830
|
+
const findings = [];
|
|
831
|
+
const text = String(command || '');
|
|
832
|
+
const hasNetwork = Boolean(segmentHasNetworkExfil(text))
|
|
833
|
+
|| /https?:\/\//i.test(text)
|
|
834
|
+
|| /\/dev\/tcp\//i.test(text);
|
|
835
|
+
if (!hasNetwork) return findings;
|
|
836
|
+
|
|
837
|
+
// stdin redirect from a secret path
|
|
838
|
+
const redirectRe = /(?:^|[\s;|&])\d*\s*<\s*([^\s;|&><]+)/g;
|
|
839
|
+
let match = redirectRe.exec(text);
|
|
840
|
+
while (match) {
|
|
841
|
+
const pathFinding = classifySecretPathToken(match[1]);
|
|
842
|
+
if (pathFinding) {
|
|
843
|
+
findings.push({
|
|
844
|
+
...pathFinding,
|
|
845
|
+
source: 'redirect_exfil',
|
|
846
|
+
reason: `${pathFinding.label} redirected into a network-bound command`,
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
match = redirectRe.exec(text);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
// Bash /dev/tcp with secret path anywhere in the same command
|
|
853
|
+
if (/\/dev\/tcp\//i.test(text)) {
|
|
854
|
+
const tokens = tokenizeCommand(text.replace(/[<>]/g, ' '));
|
|
855
|
+
for (const token of tokens) {
|
|
856
|
+
const pathFinding = classifySecretPathToken(token);
|
|
857
|
+
if (!pathFinding) continue;
|
|
858
|
+
findings.push({
|
|
859
|
+
...pathFinding,
|
|
860
|
+
source: 'dev_tcp_exfil',
|
|
861
|
+
reason: `${pathFinding.label} used with /dev/tcp network redirect`,
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// dd if=.env | curl / nc
|
|
867
|
+
const ddMatch = /\bdd\b[\s\S]*?\bif=([^\s]+)/i.exec(text);
|
|
868
|
+
if (ddMatch) {
|
|
869
|
+
const pathFinding = classifySecretPathToken(ddMatch[1]);
|
|
870
|
+
if (pathFinding && (Boolean(segmentHasNetworkExfil(text)) || /\|/.test(text) || /\/dev\/tcp\//i.test(text))) {
|
|
871
|
+
findings.push({
|
|
872
|
+
...pathFinding,
|
|
873
|
+
source: 'dd_exfil',
|
|
874
|
+
reason: `${pathFinding.label} read via dd if= toward network`,
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
return findings;
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
function detectCloudCliExfil(command, cwd) {
|
|
883
|
+
// aws s3 cp .env s3://bucket/ | gsutil cp .env gs:// | rclone copy .env remote:
|
|
884
|
+
// az storage blob upload --file .env
|
|
885
|
+
const findings = [];
|
|
886
|
+
const text = String(command || '');
|
|
887
|
+
const tokens = tokenizeCommand(text);
|
|
888
|
+
if (!tokens.length) return findings;
|
|
889
|
+
|
|
890
|
+
let index = 0;
|
|
891
|
+
while (isShellAssignment(tokens[index])) index += 1;
|
|
892
|
+
const root = path.basename(String(tokens[index] || '')).toLowerCase();
|
|
893
|
+
|
|
894
|
+
const isCloud = root === 'aws' || root === 'gsutil' || root === 'gcloud'
|
|
895
|
+
|| root === 'az' || root === 'rclone';
|
|
896
|
+
if (!isCloud) return findings;
|
|
897
|
+
|
|
898
|
+
// Require an upload-ish subcommand somewhere in the argv.
|
|
899
|
+
const hasUploadVerb = tokens.some((t) => CLOUD_UPLOAD_VERBS.has(String(t).toLowerCase()))
|
|
900
|
+
|| /\bblob\s+upload\b/i.test(text)
|
|
901
|
+
|| /\bstorage\s+cp\b/i.test(text);
|
|
902
|
+
if (!hasUploadVerb) return findings;
|
|
903
|
+
|
|
904
|
+
// Remote destination markers
|
|
905
|
+
const hasRemote = tokens.some((t) => /^(?:s3|gs|gcs|az|azure|oss):\/\//i.test(t) || /^[A-Za-z0-9_-]+:/.test(t) && !t.startsWith('/') && t.includes(':'))
|
|
906
|
+
|| /--account-name|--container|--bucket/i.test(text);
|
|
907
|
+
if (!hasRemote && root !== 'rclone') {
|
|
908
|
+
// Still flag if a secret path is an explicit --file argument for az upload
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
for (const token of tokens) {
|
|
912
|
+
if (String(token).startsWith('-') && !String(token).includes('=@') && !String(token).includes('/')) {
|
|
913
|
+
// --file=.env or --file .env handled via next token / equals form below
|
|
914
|
+
}
|
|
915
|
+
const stripped = String(token).replace(/^--?[A-Za-z0-9_-]+=/, '');
|
|
916
|
+
const pathFinding = classifySecretPathToken(stripped) || classifySecretPathToken(token);
|
|
917
|
+
if (!pathFinding) continue;
|
|
918
|
+
findings.push({
|
|
919
|
+
...pathFinding,
|
|
920
|
+
source: 'cloud_cli_exfil',
|
|
921
|
+
reason: `${pathFinding.label} referenced by cloud upload CLI (${root})`,
|
|
922
|
+
});
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
// az storage blob upload --file .env
|
|
926
|
+
const fileFlag = /(?:--file|-f)\s+([^\s]+)/i.exec(text);
|
|
927
|
+
if (fileFlag) {
|
|
928
|
+
const pathFinding = classifySecretPathToken(fileFlag[1]);
|
|
929
|
+
if (pathFinding) {
|
|
930
|
+
findings.push({
|
|
931
|
+
...pathFinding,
|
|
932
|
+
source: 'cloud_cli_exfil',
|
|
933
|
+
reason: `${pathFinding.label} passed to ${root} --file upload`,
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
return findings;
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
function detectArchivePipeExfil(command, cwd) {
|
|
942
|
+
// tar czf - .env | curl | zip - .env | nc | tar -c .ssh | …
|
|
943
|
+
const findings = [];
|
|
944
|
+
const text = String(command || '');
|
|
945
|
+
if (!/\|/.test(text) && !segmentHasNetworkExfil(text) && !/https?:\/\//i.test(text)) {
|
|
946
|
+
return findings;
|
|
947
|
+
}
|
|
948
|
+
const segments = text.split(/(?<![|])\|(?![|])/);
|
|
949
|
+
const left = segments[0] || text;
|
|
950
|
+
const archiveVerb = path.basename(String(tokenizeCommand(left)[0] || '')).toLowerCase();
|
|
951
|
+
if (!['tar', 'zip', 'gzip', 'pigz', '7z', 'rar'].includes(archiveVerb)) {
|
|
952
|
+
// Also: tar … without being first if wrappers — skip for now
|
|
953
|
+
if (!/\b(?:tar|zip)\b/i.test(left)) return findings;
|
|
954
|
+
}
|
|
955
|
+
const right = segments.slice(1).join('|') || text;
|
|
956
|
+
const networkVerb = segmentHasNetworkExfil(right) || segmentHasNetworkExfil(text);
|
|
957
|
+
if (!networkVerb && !/https?:\/\//i.test(text)) return findings;
|
|
958
|
+
|
|
959
|
+
for (const token of tokenizeCommand(left)) {
|
|
960
|
+
const pathFinding = classifySecretPathToken(token);
|
|
961
|
+
if (!pathFinding) continue;
|
|
962
|
+
findings.push({
|
|
963
|
+
...pathFinding,
|
|
964
|
+
source: 'archive_exfil',
|
|
965
|
+
reason: `${pathFinding.label} archived/streamed toward network (${networkVerb || 'url'})`,
|
|
966
|
+
});
|
|
967
|
+
}
|
|
968
|
+
// bare `.ssh` / `.aws` directories in tar
|
|
969
|
+
if (/(?:^|\s)(?:\.ssh|\.aws|\.gnupg|\.kube)(?:\s|$)/.test(left)) {
|
|
970
|
+
findings.push({
|
|
971
|
+
id: 'secret_dir_archive',
|
|
972
|
+
label: 'Secret credential directory',
|
|
973
|
+
source: 'archive_exfil',
|
|
974
|
+
reason: 'Credential directory streamed toward network',
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
return findings;
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
function detectInterpreterExfil(command, cwd) {
|
|
981
|
+
// python -c '…open(".env")…requests…' | node -e '…readFileSync(".env")…fetch…'
|
|
982
|
+
const text = String(command || '');
|
|
983
|
+
const isInterpreter = /\b(?:python3?|node|nodejs|php|ruby|perl)\b/i.test(text)
|
|
984
|
+
&& /(?:-c|-e|--eval)\b/i.test(text);
|
|
985
|
+
if (!isInterpreter) return [];
|
|
986
|
+
|
|
987
|
+
const touchesSecretPath = /\.env(?:\.[A-Za-z0-9_-]+)?|\.aws\/credentials|\.ssh\/id_|\.netrc|\.npmrc|application_default_credentials/i.test(text);
|
|
988
|
+
const touchesSecretEnv = /(?:API_KEY|OPENAI_API_KEY|AWS_SECRET|GITHUB_TOKEN|STRIPE_SECRET|os\.environ|process\.env)/i.test(text);
|
|
989
|
+
const touchesNetwork = /https?:\/\/|requests\.|urllib|fetch\(|http\.request|httpx\.|aiohttp|socket\.|urlopen/i.test(text);
|
|
990
|
+
if (!(touchesSecretPath || touchesSecretEnv) || !touchesNetwork) return [];
|
|
991
|
+
|
|
992
|
+
return [{
|
|
993
|
+
id: 'interpreter_exfil',
|
|
994
|
+
label: 'Interpreter secret exfiltration',
|
|
995
|
+
source: 'interpreter_exfil',
|
|
996
|
+
reason: 'Inline interpreter code appears to read secret material and perform network I/O',
|
|
997
|
+
}];
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
function detectStructuralExfiltration(command, cwd) {
|
|
1001
|
+
return uniqueFindings([
|
|
1002
|
+
...detectCommandSubstitutionExfil(command, cwd),
|
|
1003
|
+
...detectPipelineExfil(command, cwd),
|
|
1004
|
+
...detectScpRsyncExfil(command, cwd),
|
|
1005
|
+
...detectEnvVarNetworkExfil(command),
|
|
1006
|
+
...detectRedirectExfil(command, cwd),
|
|
1007
|
+
...detectCloudCliExfil(command, cwd),
|
|
1008
|
+
...detectArchivePipeExfil(command, cwd),
|
|
1009
|
+
...detectInterpreterExfil(command, cwd),
|
|
1010
|
+
]);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
628
1013
|
function scanBashCommand(command, options = {}) {
|
|
629
1014
|
const cwd = options.cwd || process.cwd();
|
|
630
1015
|
const inlineScan = scanText(command, { provider: options.provider, source: 'command' });
|
|
@@ -636,6 +1021,12 @@ function scanBashCommand(command, options = {}) {
|
|
|
636
1021
|
: scanOutboundCommandFiles(command, cwd, options.provider);
|
|
637
1022
|
findings.push(...outboundScan.findings);
|
|
638
1023
|
|
|
1024
|
+
// Structural vectors: secret never appears as a literal in the command text
|
|
1025
|
+
// (pipe, command substitution, scp, env-var → network).
|
|
1026
|
+
if (inlineScan.provider !== 'off') {
|
|
1027
|
+
findings.push(...detectStructuralExfiltration(command, cwd));
|
|
1028
|
+
}
|
|
1029
|
+
|
|
639
1030
|
const uniqueFileHashes = [...new Set(outboundScan.fileHashes)];
|
|
640
1031
|
|
|
641
1032
|
return {
|
|
@@ -399,7 +399,13 @@ function readRunManifests() {
|
|
|
399
399
|
async function runSelfDistill({ dryRun = false, limit = 20, model } = {}) {
|
|
400
400
|
const startedAt = new Date().toISOString();
|
|
401
401
|
const logPaths = discoverConversationLogs({ limit });
|
|
402
|
-
|
|
402
|
+
// Ask llm-client which provider is actually live rather than testing for one
|
|
403
|
+
// vendor's key. Checking ANTHROPIC_API_KEY directly meant an operator running
|
|
404
|
+
// a configured gateway got heuristic distillation while a working model sat
|
|
405
|
+
// one call away — and the manifest said 'heuristic' without explaining why.
|
|
406
|
+
const { describeInferenceAvailability } = require('./llm-client');
|
|
407
|
+
const inference = describeInferenceAvailability();
|
|
408
|
+
const hasApiKey = inference.available;
|
|
403
409
|
const analysisMode = hasApiKey ? 'llm' : 'heuristic';
|
|
404
410
|
|
|
405
411
|
const allLessons = [];
|
|
@@ -20,6 +20,25 @@ const DEFAULT_CHECKS = [
|
|
|
20
20
|
{ name: 'prove_automation', command: ['npm', 'run', 'prove:automation'], timeoutMs: 10 * 60_000, useTempProofDir: true },
|
|
21
21
|
{ name: 'prove_data_pipeline', command: ['npm', 'run', 'prove:data-pipeline'], timeoutMs: 10 * 60_000, useTempProofDir: true },
|
|
22
22
|
{ name: 'prove_tessl', command: ['npm', 'run', 'prove:tessl'], timeoutMs: 10 * 60_000, useTempProofDir: true },
|
|
23
|
+
// Antithesis-style reliability explorer (fixed seed, cheap, deterministic)
|
|
24
|
+
{
|
|
25
|
+
name: 'prove_reliability',
|
|
26
|
+
command: ['npm', 'run', 'prove:reliability'],
|
|
27
|
+
timeoutMs: 3 * 60_000,
|
|
28
|
+
useTempProofDir: true,
|
|
29
|
+
},
|
|
30
|
+
// Glama/MCP registry start contract — cheap pin vs recurring "Build failed for ThumbGate"
|
|
31
|
+
{
|
|
32
|
+
name: 'prove_glama_mcp',
|
|
33
|
+
command: ['npm', 'run', 'prove:glama-mcp'],
|
|
34
|
+
timeoutMs: 60_000,
|
|
35
|
+
useTempProofDir: true,
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
name: 'lesson_embedding_index',
|
|
39
|
+
command: ['node', 'scripts/lesson-embedding-maintenance.js', '--json'],
|
|
40
|
+
timeoutMs: 60_000,
|
|
41
|
+
},
|
|
23
42
|
];
|
|
24
43
|
|
|
25
44
|
function runCommand(command, {
|
|
@@ -60,6 +79,12 @@ function createCheckEnvironment(check) {
|
|
|
60
79
|
if (check.name === 'prove_automation') {
|
|
61
80
|
environment.THUMBGATE_AUTOMATION_PROOF_DIR = proofDir;
|
|
62
81
|
}
|
|
82
|
+
if (check.name === 'prove_reliability') {
|
|
83
|
+
environment.THUMBGATE_RELIABILITY_PROOF_DIR = proofDir;
|
|
84
|
+
// Pin cheap deterministic exploration for health checks
|
|
85
|
+
environment.THUMBGATE_RELIABILITY_SEED = environment.THUMBGATE_RELIABILITY_SEED || '42';
|
|
86
|
+
environment.THUMBGATE_RELIABILITY_ITERATIONS = environment.THUMBGATE_RELIABILITY_ITERATIONS || '8';
|
|
87
|
+
}
|
|
63
88
|
cleanup = () => {
|
|
64
89
|
fs.rmSync(proofDir, { recursive: true, force: true });
|
|
65
90
|
};
|