ship-safe 6.1.1 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +735 -641
- package/cli/agents/api-fuzzer.js +345 -345
- package/cli/agents/auth-bypass-agent.js +348 -348
- package/cli/agents/base-agent.js +272 -272
- package/cli/agents/cicd-scanner.js +236 -201
- package/cli/agents/config-auditor.js +521 -521
- package/cli/agents/deep-analyzer.js +6 -2
- package/cli/agents/git-history-scanner.js +170 -170
- package/cli/agents/html-reporter.js +568 -568
- package/cli/agents/index.js +84 -84
- package/cli/agents/injection-tester.js +500 -500
- package/cli/agents/llm-redteam.js +251 -251
- package/cli/agents/mobile-scanner.js +231 -231
- package/cli/agents/orchestrator.js +322 -322
- package/cli/agents/pii-compliance-agent.js +301 -301
- package/cli/agents/scoring-engine.js +248 -248
- package/cli/agents/supabase-rls-agent.js +154 -154
- package/cli/agents/supply-chain-agent.js +650 -507
- package/cli/bin/ship-safe.js +452 -426
- package/cli/commands/agent.js +608 -608
- package/cli/commands/audit.js +986 -980
- package/cli/commands/baseline.js +193 -193
- package/cli/commands/ci.js +342 -342
- package/cli/commands/deps.js +516 -516
- package/cli/commands/doctor.js +159 -159
- package/cli/commands/fix.js +218 -218
- package/cli/commands/hooks.js +268 -0
- package/cli/commands/init.js +407 -407
- package/cli/commands/mcp.js +304 -304
- package/cli/commands/red-team.js +7 -1
- package/cli/commands/remediate.js +798 -798
- package/cli/commands/rotate.js +571 -571
- package/cli/commands/scan.js +569 -569
- package/cli/commands/score.js +449 -449
- package/cli/commands/watch.js +281 -281
- package/cli/hooks/patterns.js +313 -0
- package/cli/hooks/post-tool-use.js +140 -0
- package/cli/hooks/pre-tool-use.js +186 -0
- package/cli/index.js +73 -69
- package/cli/providers/llm-provider.js +397 -287
- package/cli/utils/autofix-rules.js +74 -74
- package/cli/utils/cache-manager.js +311 -311
- package/cli/utils/output.js +230 -230
- package/cli/utils/patterns.js +1121 -1121
- package/cli/utils/pdf-generator.js +94 -94
- package/package.json +69 -69
- package/configs/supabase/rls-templates.sql +0 -242
|
@@ -1,251 +1,251 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* LLMRedTeam Agent
|
|
3
|
-
* =================
|
|
4
|
-
*
|
|
5
|
-
* AI/LLM security testing based on OWASP LLM Top 10 2025.
|
|
6
|
-
* Detects prompt injection vulnerabilities, system prompt leakage,
|
|
7
|
-
* unsafe LLM output handling, excessive agency, and more.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import path from 'path';
|
|
11
|
-
import { BaseAgent } from './base-agent.js';
|
|
12
|
-
|
|
13
|
-
const PATTERNS = [
|
|
14
|
-
// ── LLM01: Prompt Injection ────────────────────────────────────────────────
|
|
15
|
-
{
|
|
16
|
-
rule: 'LLM_PROMPT_INJECTION_NO_SANITIZE',
|
|
17
|
-
title: 'Prompt Injection: No Input Sanitization',
|
|
18
|
-
regex: /(?:messages|prompt|content)\s*[:=]\s*(?:`[^`]*\$\{(?:req\.|request\.|body|query|params|input|user)|[^\n]*\+\s*(?:req\.|request\.|body|query|params|input|user))/g,
|
|
19
|
-
severity: 'high',
|
|
20
|
-
cwe: 'CWE-77',
|
|
21
|
-
owasp: 'LLM01',
|
|
22
|
-
description: 'User input concatenated directly into LLM prompt without sanitization enables prompt injection.',
|
|
23
|
-
fix: 'Sanitize user input, use structured messages, separate system/user content clearly',
|
|
24
|
-
},
|
|
25
|
-
{
|
|
26
|
-
rule: 'LLM_SYSTEM_USER_CONCAT',
|
|
27
|
-
title: 'Prompt Injection: System + User Concatenation',
|
|
28
|
-
regex: /(?:system|systemPrompt|system_prompt)\s*[:=][^\n]*(?:\+\s*(?:user|input|query|message)|`[^`]*\$\{)/g,
|
|
29
|
-
severity: 'critical',
|
|
30
|
-
cwe: 'CWE-77',
|
|
31
|
-
owasp: 'LLM01',
|
|
32
|
-
description: 'System prompt concatenated with user input. User can override system instructions.',
|
|
33
|
-
fix: 'Use separate message roles: [{role:"system", content: systemPrompt}, {role:"user", content: userInput}]',
|
|
34
|
-
},
|
|
35
|
-
|
|
36
|
-
// ── LLM02: Sensitive Information Disclosure ────────────────────────────────
|
|
37
|
-
{
|
|
38
|
-
rule: 'LLM_SECRET_IN_PROMPT',
|
|
39
|
-
title: 'Sensitive Data in LLM Prompt',
|
|
40
|
-
regex: /(?:system|prompt|content)\s*[:=][^\n]*(?:API_KEY|api_key|SECRET|PASSWORD|TOKEN|PRIVATE_KEY|DATABASE_URL)/g,
|
|
41
|
-
severity: 'critical',
|
|
42
|
-
cwe: 'CWE-200',
|
|
43
|
-
owasp: 'LLM02',
|
|
44
|
-
description: 'Sensitive data (secrets, keys) included in LLM prompt. Data may be logged or leaked.',
|
|
45
|
-
fix: 'Never include real credentials in prompts. Use placeholder references instead.',
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
rule: 'LLM_NO_OUTPUT_FILTER',
|
|
49
|
-
title: 'LLM Output Without Filtering',
|
|
50
|
-
regex: /(?:completion|response|result|output)(?:\.\w+)*\.(?:content|text|message)\s*(?:\)|;)/g,
|
|
51
|
-
severity: 'medium',
|
|
52
|
-
cwe: 'CWE-200',
|
|
53
|
-
owasp: 'LLM02',
|
|
54
|
-
confidence: 'low',
|
|
55
|
-
description: 'LLM output used directly without filtering. May contain sensitive info or hallucinations.',
|
|
56
|
-
fix: 'Filter LLM output before displaying: remove PII, validate against expected format',
|
|
57
|
-
},
|
|
58
|
-
|
|
59
|
-
// ── LLM05: Improper Output Handling ────────────────────────────────────────
|
|
60
|
-
{
|
|
61
|
-
rule: 'LLM_OUTPUT_TO_EVAL',
|
|
62
|
-
title: 'LLM Output to eval()/Function()',
|
|
63
|
-
regex: /eval\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
64
|
-
severity: 'critical',
|
|
65
|
-
cwe: 'CWE-94',
|
|
66
|
-
owasp: 'LLM05',
|
|
67
|
-
description: 'LLM output passed to eval() enables arbitrary code execution via prompt injection.',
|
|
68
|
-
fix: 'Never eval() LLM output. Parse as JSON with try/catch, or use a sandboxed interpreter.',
|
|
69
|
-
},
|
|
70
|
-
{
|
|
71
|
-
rule: 'LLM_OUTPUT_TO_SQL',
|
|
72
|
-
title: 'LLM Output in SQL Query',
|
|
73
|
-
regex: /(?:query|execute|raw)\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
74
|
-
severity: 'critical',
|
|
75
|
-
cwe: 'CWE-89',
|
|
76
|
-
owasp: 'LLM05',
|
|
77
|
-
description: 'LLM-generated text used in SQL query. Attacker can inject SQL via prompt injection.',
|
|
78
|
-
fix: 'Never use LLM output in raw SQL. Validate against expected query patterns.',
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
rule: 'LLM_OUTPUT_TO_HTML',
|
|
82
|
-
title: 'LLM Output Rendered as HTML',
|
|
83
|
-
regex: /(?:innerHTML|dangerouslySetInnerHTML|v-html)\s*=\s*(?:.*(?:completion|response|result|output|generated|llm|ai|gpt|claude))/gi,
|
|
84
|
-
severity: 'high',
|
|
85
|
-
cwe: 'CWE-79',
|
|
86
|
-
owasp: 'LLM05',
|
|
87
|
-
description: 'LLM output rendered as unescaped HTML enables XSS via prompt injection.',
|
|
88
|
-
fix: 'Render LLM output as text, or sanitize with DOMPurify before HTML rendering.',
|
|
89
|
-
},
|
|
90
|
-
{
|
|
91
|
-
rule: 'LLM_OUTPUT_TO_SHELL',
|
|
92
|
-
title: 'LLM Output in Shell Command',
|
|
93
|
-
regex: /(?:exec|spawn|system|popen)\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
94
|
-
severity: 'critical',
|
|
95
|
-
cwe: 'CWE-78',
|
|
96
|
-
owasp: 'LLM05',
|
|
97
|
-
description: 'LLM output used in shell command enables RCE via prompt injection.',
|
|
98
|
-
fix: 'Never pass LLM output to shell. Use a strict allowlist of allowed commands.',
|
|
99
|
-
},
|
|
100
|
-
|
|
101
|
-
// ── LLM06: Excessive Agency ────────────────────────────────────────────────
|
|
102
|
-
{
|
|
103
|
-
rule: 'LLM_TOOL_NO_CONFIRM',
|
|
104
|
-
title: 'LLM Tool Use Without Confirmation',
|
|
105
|
-
regex: /(?:tools|functions|function_call)\s*[:=]\s*\[.*(?:write|delete|update|create|send|execute|deploy|modify)/gi,
|
|
106
|
-
severity: 'high',
|
|
107
|
-
cwe: 'CWE-862',
|
|
108
|
-
owasp: 'LLM06',
|
|
109
|
-
confidence: 'medium',
|
|
110
|
-
description: 'LLM given tools with side effects (write/delete/send) without human confirmation.',
|
|
111
|
-
fix: 'Require human approval for destructive actions. Implement an approval workflow.',
|
|
112
|
-
},
|
|
113
|
-
{
|
|
114
|
-
rule: 'LLM_DB_WRITE_ACCESS',
|
|
115
|
-
title: 'LLM Has Database Write Access',
|
|
116
|
-
regex: /(?:tool|function).*(?:INSERT|UPDATE|DELETE|DROP|CREATE|ALTER).*(?:sql|query|database|db)/gi,
|
|
117
|
-
severity: 'critical',
|
|
118
|
-
cwe: 'CWE-862',
|
|
119
|
-
owasp: 'LLM06',
|
|
120
|
-
description: 'LLM can write to database. Prompt injection could corrupt or destroy data.',
|
|
121
|
-
fix: 'Give LLM read-only database access. Require human approval for writes.',
|
|
122
|
-
},
|
|
123
|
-
{
|
|
124
|
-
rule: 'LLM_FILE_WRITE',
|
|
125
|
-
title: 'LLM Has File System Write Access',
|
|
126
|
-
regex: /(?:tool|function).*(?:writeFile|fs\.write|unlink|rmdir|mkdir)/gi,
|
|
127
|
-
severity: 'critical',
|
|
128
|
-
cwe: 'CWE-862',
|
|
129
|
-
owasp: 'LLM06',
|
|
130
|
-
description: 'LLM can write/delete files. Prompt injection could modify or destroy files.',
|
|
131
|
-
fix: 'Restrict LLM file access to a sandboxed directory with read-only permissions.',
|
|
132
|
-
},
|
|
133
|
-
|
|
134
|
-
// ── LLM07: System Prompt Leakage ───────────────────────────────────────────
|
|
135
|
-
{
|
|
136
|
-
rule: 'LLM_SYSTEM_PROMPT_CLIENT',
|
|
137
|
-
title: 'System Prompt Exposed to Client',
|
|
138
|
-
regex: /(?:systemPrompt|system_prompt|SYSTEM_PROMPT)\s*[:=]\s*["'`]/g,
|
|
139
|
-
severity: 'high',
|
|
140
|
-
cwe: 'CWE-200',
|
|
141
|
-
owasp: 'LLM07',
|
|
142
|
-
confidence: 'medium',
|
|
143
|
-
description: 'System prompt hardcoded in code. If client-side, users can extract it.',
|
|
144
|
-
fix: 'Keep system prompts server-side only. Load from environment variables or config.',
|
|
145
|
-
},
|
|
146
|
-
|
|
147
|
-
// ── LLM10: Unbounded Consumption ───────────────────────────────────────────
|
|
148
|
-
{
|
|
149
|
-
rule: 'LLM_NO_TOKEN_LIMIT',
|
|
150
|
-
title: 'LLM Call Without Token Limit',
|
|
151
|
-
regex: /(?:openai|anthropic|ai)\.\w+\.create\s*\(\s*\{(?![\s\S]*max_tokens)[\s\S]*?\}/g,
|
|
152
|
-
severity: 'medium',
|
|
153
|
-
cwe: 'CWE-770',
|
|
154
|
-
owasp: 'LLM10',
|
|
155
|
-
confidence: 'low',
|
|
156
|
-
description: 'LLM API call without max_tokens limit. Could generate excessive output and costs.',
|
|
157
|
-
fix: 'Set max_tokens in API call to limit response size and costs',
|
|
158
|
-
},
|
|
159
|
-
{
|
|
160
|
-
rule: 'LLM_NO_RATE_LIMIT',
|
|
161
|
-
title: 'LLM Endpoint Without Rate Limiting',
|
|
162
|
-
regex: /(?:\/api\/.*(?:chat|complete|generate|ai|llm|gpt|claude)|\/chat|\/generate)\s*['"]/gi,
|
|
163
|
-
severity: 'medium',
|
|
164
|
-
cwe: 'CWE-770',
|
|
165
|
-
owasp: 'LLM10',
|
|
166
|
-
confidence: 'low',
|
|
167
|
-
description: 'AI endpoint without rate limiting. Users could rack up API costs.',
|
|
168
|
-
fix: 'Add rate limiting per user: express-rate-limit, @upstash/ratelimit, etc.',
|
|
169
|
-
},
|
|
170
|
-
{
|
|
171
|
-
rule: 'LLM_NO_COST_LIMIT',
|
|
172
|
-
title: 'LLM Usage Without Cost Controls',
|
|
173
|
-
regex: /(?:OPENAI|ANTHROPIC|AI)_(?:API_KEY|KEY).*(?!(?:budget|limit|cap|max_cost|spending))/gi,
|
|
174
|
-
severity: 'medium',
|
|
175
|
-
cwe: 'CWE-770',
|
|
176
|
-
owasp: 'LLM10',
|
|
177
|
-
confidence: 'low',
|
|
178
|
-
description: 'AI API usage without cost controls. Set spending limits on your provider dashboard.',
|
|
179
|
-
fix: 'Configure spending limits in OpenAI/Anthropic dashboard. Add per-user token budgets.',
|
|
180
|
-
},
|
|
181
|
-
|
|
182
|
-
// ── LLM03: Supply Chain ────────────────────────────────────────────────────
|
|
183
|
-
{
|
|
184
|
-
rule: 'LLM_UNVERIFIED_MODEL',
|
|
185
|
-
title: 'Unverified Model Download',
|
|
186
|
-
regex: /(?:from_pretrained|AutoModel|pipeline)\s*\(\s*["'][^"']+\/[^"']+["']/g,
|
|
187
|
-
severity: 'medium',
|
|
188
|
-
cwe: 'CWE-829',
|
|
189
|
-
owasp: 'LLM03',
|
|
190
|
-
confidence: 'low',
|
|
191
|
-
description: 'Loading model from Hugging Face without verification. Model could contain backdoors.',
|
|
192
|
-
fix: 'Verify model hash, use models from trusted organizations, scan for malicious code',
|
|
193
|
-
},
|
|
194
|
-
|
|
195
|
-
// ── LLM08: Vector/Embedding Weaknesses ─────────────────────────────────────
|
|
196
|
-
{
|
|
197
|
-
rule: 'LLM_RAG_NO_VALIDATION',
|
|
198
|
-
title: 'RAG Pipeline Without Input Validation',
|
|
199
|
-
regex: /(?:embed|embedding|vector|similarity_search|query)\s*\(\s*(?:req\.|request\.|body|query|params|input|user)/g,
|
|
200
|
-
severity: 'medium',
|
|
201
|
-
cwe: 'CWE-20',
|
|
202
|
-
owasp: 'LLM08',
|
|
203
|
-
description: 'User input passed directly to vector search/embedding without validation.',
|
|
204
|
-
fix: 'Validate and sanitize input before embedding. Limit query length.',
|
|
205
|
-
},
|
|
206
|
-
{
|
|
207
|
-
rule: 'LLM_RAG_NO_ACCESS_CONTROL',
|
|
208
|
-
title: 'RAG Without Access Control',
|
|
209
|
-
regex: /(?:pinecone|chroma|weaviate|qdrant|milvus).*(?:query|search|similarity)\s*\(/g,
|
|
210
|
-
severity: 'medium',
|
|
211
|
-
cwe: 'CWE-862',
|
|
212
|
-
owasp: 'LLM08',
|
|
213
|
-
confidence: 'low',
|
|
214
|
-
description: 'Vector database query without access control. Users may access other users\' data.',
|
|
215
|
-
fix: 'Add namespace/tenant filtering: filter by userId in vector DB queries',
|
|
216
|
-
},
|
|
217
|
-
|
|
218
|
-
// ── Prompt Injection Patterns (content-level detection) ────────────────────
|
|
219
|
-
{
|
|
220
|
-
rule: 'PROMPT_INJECTION_PATTERN',
|
|
221
|
-
title: 'Known Prompt Injection Pattern',
|
|
222
|
-
regex: /(?:ignore\s+(?:all\s+)?previous\s+instructions|disregard\s+(?:all\s+)?(?:previous|prior)|you\s+are\s+now\s+DAN|system\s*prompt|jailbreak|bypass\s+(?:your|the)\s+(?:rules|instructions|guidelines))/gi,
|
|
223
|
-
severity: 'high',
|
|
224
|
-
cwe: 'CWE-77',
|
|
225
|
-
owasp: 'LLM01',
|
|
226
|
-
description: 'Known prompt injection pattern detected in code. Ensure this is for testing only.',
|
|
227
|
-
fix: 'If in test data, add # ship-safe-ignore. If in user-facing code, add input filtering.',
|
|
228
|
-
},
|
|
229
|
-
];
|
|
230
|
-
|
|
231
|
-
export class LLMRedTeam extends BaseAgent {
|
|
232
|
-
constructor() {
|
|
233
|
-
super('LLMRedTeam', 'AI/LLM security audit based on OWASP LLM Top 10', 'llm');
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
async analyze(context) {
|
|
237
|
-
const { files } = context;
|
|
238
|
-
const codeFiles = files.filter(f => {
|
|
239
|
-
const ext = path.extname(f).toLowerCase();
|
|
240
|
-
return ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.py', '.rb'].includes(ext);
|
|
241
|
-
});
|
|
242
|
-
|
|
243
|
-
let findings = [];
|
|
244
|
-
for (const file of codeFiles) {
|
|
245
|
-
findings = findings.concat(this.scanFileWithPatterns(file, PATTERNS));
|
|
246
|
-
}
|
|
247
|
-
return findings;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
export default LLMRedTeam;
|
|
1
|
+
/**
|
|
2
|
+
* LLMRedTeam Agent
|
|
3
|
+
* =================
|
|
4
|
+
*
|
|
5
|
+
* AI/LLM security testing based on OWASP LLM Top 10 2025.
|
|
6
|
+
* Detects prompt injection vulnerabilities, system prompt leakage,
|
|
7
|
+
* unsafe LLM output handling, excessive agency, and more.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import path from 'path';
|
|
11
|
+
import { BaseAgent } from './base-agent.js';
|
|
12
|
+
|
|
13
|
+
const PATTERNS = [
|
|
14
|
+
// ── LLM01: Prompt Injection ────────────────────────────────────────────────
|
|
15
|
+
{
|
|
16
|
+
rule: 'LLM_PROMPT_INJECTION_NO_SANITIZE',
|
|
17
|
+
title: 'Prompt Injection: No Input Sanitization',
|
|
18
|
+
regex: /(?:messages|prompt|content)\s*[:=]\s*(?:`[^`]*\$\{(?:req\.|request\.|body|query|params|input|user)|[^\n]*\+\s*(?:req\.|request\.|body|query|params|input|user))/g,
|
|
19
|
+
severity: 'high',
|
|
20
|
+
cwe: 'CWE-77',
|
|
21
|
+
owasp: 'LLM01',
|
|
22
|
+
description: 'User input concatenated directly into LLM prompt without sanitization enables prompt injection.',
|
|
23
|
+
fix: 'Sanitize user input, use structured messages, separate system/user content clearly',
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
rule: 'LLM_SYSTEM_USER_CONCAT',
|
|
27
|
+
title: 'Prompt Injection: System + User Concatenation',
|
|
28
|
+
regex: /(?:system|systemPrompt|system_prompt)\s*[:=][^\n]*(?:\+\s*(?:user|input|query|message)|`[^`]*\$\{)/g,
|
|
29
|
+
severity: 'critical',
|
|
30
|
+
cwe: 'CWE-77',
|
|
31
|
+
owasp: 'LLM01',
|
|
32
|
+
description: 'System prompt concatenated with user input. User can override system instructions.',
|
|
33
|
+
fix: 'Use separate message roles: [{role:"system", content: systemPrompt}, {role:"user", content: userInput}]',
|
|
34
|
+
},
|
|
35
|
+
|
|
36
|
+
// ── LLM02: Sensitive Information Disclosure ────────────────────────────────
|
|
37
|
+
{
|
|
38
|
+
rule: 'LLM_SECRET_IN_PROMPT',
|
|
39
|
+
title: 'Sensitive Data in LLM Prompt',
|
|
40
|
+
regex: /(?:system|prompt|content)\s*[:=][^\n]*(?:API_KEY|api_key|SECRET|PASSWORD|TOKEN|PRIVATE_KEY|DATABASE_URL)/g,
|
|
41
|
+
severity: 'critical',
|
|
42
|
+
cwe: 'CWE-200',
|
|
43
|
+
owasp: 'LLM02',
|
|
44
|
+
description: 'Sensitive data (secrets, keys) included in LLM prompt. Data may be logged or leaked.',
|
|
45
|
+
fix: 'Never include real credentials in prompts. Use placeholder references instead.',
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
rule: 'LLM_NO_OUTPUT_FILTER',
|
|
49
|
+
title: 'LLM Output Without Filtering',
|
|
50
|
+
regex: /(?:completion|response|result|output)(?:\.\w+)*\.(?:content|text|message)\s*(?:\)|;)/g,
|
|
51
|
+
severity: 'medium',
|
|
52
|
+
cwe: 'CWE-200',
|
|
53
|
+
owasp: 'LLM02',
|
|
54
|
+
confidence: 'low',
|
|
55
|
+
description: 'LLM output used directly without filtering. May contain sensitive info or hallucinations.',
|
|
56
|
+
fix: 'Filter LLM output before displaying: remove PII, validate against expected format',
|
|
57
|
+
},
|
|
58
|
+
|
|
59
|
+
// ── LLM05: Improper Output Handling ────────────────────────────────────────
|
|
60
|
+
{
|
|
61
|
+
rule: 'LLM_OUTPUT_TO_EVAL',
|
|
62
|
+
title: 'LLM Output to eval()/Function()',
|
|
63
|
+
regex: /eval\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
64
|
+
severity: 'critical',
|
|
65
|
+
cwe: 'CWE-94',
|
|
66
|
+
owasp: 'LLM05',
|
|
67
|
+
description: 'LLM output passed to eval() enables arbitrary code execution via prompt injection.',
|
|
68
|
+
fix: 'Never eval() LLM output. Parse as JSON with try/catch, or use a sandboxed interpreter.',
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
rule: 'LLM_OUTPUT_TO_SQL',
|
|
72
|
+
title: 'LLM Output in SQL Query',
|
|
73
|
+
regex: /(?:query|execute|raw)\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
74
|
+
severity: 'critical',
|
|
75
|
+
cwe: 'CWE-89',
|
|
76
|
+
owasp: 'LLM05',
|
|
77
|
+
description: 'LLM-generated text used in SQL query. Attacker can inject SQL via prompt injection.',
|
|
78
|
+
fix: 'Never use LLM output in raw SQL. Validate against expected query patterns.',
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
rule: 'LLM_OUTPUT_TO_HTML',
|
|
82
|
+
title: 'LLM Output Rendered as HTML',
|
|
83
|
+
regex: /(?:innerHTML|dangerouslySetInnerHTML|v-html)\s*=\s*(?:.*(?:completion|response|result|output|generated|llm|ai|gpt|claude))/gi,
|
|
84
|
+
severity: 'high',
|
|
85
|
+
cwe: 'CWE-79',
|
|
86
|
+
owasp: 'LLM05',
|
|
87
|
+
description: 'LLM output rendered as unescaped HTML enables XSS via prompt injection.',
|
|
88
|
+
fix: 'Render LLM output as text, or sanitize with DOMPurify before HTML rendering.',
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
rule: 'LLM_OUTPUT_TO_SHELL',
|
|
92
|
+
title: 'LLM Output in Shell Command',
|
|
93
|
+
regex: /(?:exec|spawn|system|popen)\s*\(\s*(?:completion|response|result|output|generated|llm|ai|gpt|claude)/gi,
|
|
94
|
+
severity: 'critical',
|
|
95
|
+
cwe: 'CWE-78',
|
|
96
|
+
owasp: 'LLM05',
|
|
97
|
+
description: 'LLM output used in shell command enables RCE via prompt injection.',
|
|
98
|
+
fix: 'Never pass LLM output to shell. Use a strict allowlist of allowed commands.',
|
|
99
|
+
},
|
|
100
|
+
|
|
101
|
+
// ── LLM06: Excessive Agency ────────────────────────────────────────────────
|
|
102
|
+
{
|
|
103
|
+
rule: 'LLM_TOOL_NO_CONFIRM',
|
|
104
|
+
title: 'LLM Tool Use Without Confirmation',
|
|
105
|
+
regex: /(?:tools|functions|function_call)\s*[:=]\s*\[.*(?:write|delete|update|create|send|execute|deploy|modify)/gi,
|
|
106
|
+
severity: 'high',
|
|
107
|
+
cwe: 'CWE-862',
|
|
108
|
+
owasp: 'LLM06',
|
|
109
|
+
confidence: 'medium',
|
|
110
|
+
description: 'LLM given tools with side effects (write/delete/send) without human confirmation.',
|
|
111
|
+
fix: 'Require human approval for destructive actions. Implement an approval workflow.',
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
rule: 'LLM_DB_WRITE_ACCESS',
|
|
115
|
+
title: 'LLM Has Database Write Access',
|
|
116
|
+
regex: /(?:tool|function).*(?:INSERT|UPDATE|DELETE|DROP|CREATE|ALTER).*(?:sql|query|database|db)/gi,
|
|
117
|
+
severity: 'critical',
|
|
118
|
+
cwe: 'CWE-862',
|
|
119
|
+
owasp: 'LLM06',
|
|
120
|
+
description: 'LLM can write to database. Prompt injection could corrupt or destroy data.',
|
|
121
|
+
fix: 'Give LLM read-only database access. Require human approval for writes.',
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
rule: 'LLM_FILE_WRITE',
|
|
125
|
+
title: 'LLM Has File System Write Access',
|
|
126
|
+
regex: /(?:tool|function).*(?:writeFile|fs\.write|unlink|rmdir|mkdir)/gi,
|
|
127
|
+
severity: 'critical',
|
|
128
|
+
cwe: 'CWE-862',
|
|
129
|
+
owasp: 'LLM06',
|
|
130
|
+
description: 'LLM can write/delete files. Prompt injection could modify or destroy files.',
|
|
131
|
+
fix: 'Restrict LLM file access to a sandboxed directory with read-only permissions.',
|
|
132
|
+
},
|
|
133
|
+
|
|
134
|
+
// ── LLM07: System Prompt Leakage ───────────────────────────────────────────
|
|
135
|
+
{
|
|
136
|
+
rule: 'LLM_SYSTEM_PROMPT_CLIENT',
|
|
137
|
+
title: 'System Prompt Exposed to Client',
|
|
138
|
+
regex: /(?:systemPrompt|system_prompt|SYSTEM_PROMPT)\s*[:=]\s*["'`]/g,
|
|
139
|
+
severity: 'high',
|
|
140
|
+
cwe: 'CWE-200',
|
|
141
|
+
owasp: 'LLM07',
|
|
142
|
+
confidence: 'medium',
|
|
143
|
+
description: 'System prompt hardcoded in code. If client-side, users can extract it.',
|
|
144
|
+
fix: 'Keep system prompts server-side only. Load from environment variables or config.',
|
|
145
|
+
},
|
|
146
|
+
|
|
147
|
+
// ── LLM10: Unbounded Consumption ───────────────────────────────────────────
|
|
148
|
+
{
|
|
149
|
+
rule: 'LLM_NO_TOKEN_LIMIT',
|
|
150
|
+
title: 'LLM Call Without Token Limit',
|
|
151
|
+
regex: /(?:openai|anthropic|ai)\.\w+\.create\s*\(\s*\{(?![\s\S]*max_tokens)[\s\S]*?\}/g,
|
|
152
|
+
severity: 'medium',
|
|
153
|
+
cwe: 'CWE-770',
|
|
154
|
+
owasp: 'LLM10',
|
|
155
|
+
confidence: 'low',
|
|
156
|
+
description: 'LLM API call without max_tokens limit. Could generate excessive output and costs.',
|
|
157
|
+
fix: 'Set max_tokens in API call to limit response size and costs',
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
rule: 'LLM_NO_RATE_LIMIT',
|
|
161
|
+
title: 'LLM Endpoint Without Rate Limiting',
|
|
162
|
+
regex: /(?:\/api\/.*(?:chat|complete|generate|ai|llm|gpt|claude)|\/chat|\/generate)\s*['"]/gi,
|
|
163
|
+
severity: 'medium',
|
|
164
|
+
cwe: 'CWE-770',
|
|
165
|
+
owasp: 'LLM10',
|
|
166
|
+
confidence: 'low',
|
|
167
|
+
description: 'AI endpoint without rate limiting. Users could rack up API costs.',
|
|
168
|
+
fix: 'Add rate limiting per user: express-rate-limit, @upstash/ratelimit, etc.',
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
rule: 'LLM_NO_COST_LIMIT',
|
|
172
|
+
title: 'LLM Usage Without Cost Controls',
|
|
173
|
+
regex: /(?:OPENAI|ANTHROPIC|AI)_(?:API_KEY|KEY).*(?!(?:budget|limit|cap|max_cost|spending))/gi,
|
|
174
|
+
severity: 'medium',
|
|
175
|
+
cwe: 'CWE-770',
|
|
176
|
+
owasp: 'LLM10',
|
|
177
|
+
confidence: 'low',
|
|
178
|
+
description: 'AI API usage without cost controls. Set spending limits on your provider dashboard.',
|
|
179
|
+
fix: 'Configure spending limits in OpenAI/Anthropic dashboard. Add per-user token budgets.',
|
|
180
|
+
},
|
|
181
|
+
|
|
182
|
+
// ── LLM03: Supply Chain ────────────────────────────────────────────────────
|
|
183
|
+
{
|
|
184
|
+
rule: 'LLM_UNVERIFIED_MODEL',
|
|
185
|
+
title: 'Unverified Model Download',
|
|
186
|
+
regex: /(?:from_pretrained|AutoModel|pipeline)\s*\(\s*["'][^"']+\/[^"']+["']/g,
|
|
187
|
+
severity: 'medium',
|
|
188
|
+
cwe: 'CWE-829',
|
|
189
|
+
owasp: 'LLM03',
|
|
190
|
+
confidence: 'low',
|
|
191
|
+
description: 'Loading model from Hugging Face without verification. Model could contain backdoors.',
|
|
192
|
+
fix: 'Verify model hash, use models from trusted organizations, scan for malicious code',
|
|
193
|
+
},
|
|
194
|
+
|
|
195
|
+
// ── LLM08: Vector/Embedding Weaknesses ─────────────────────────────────────
|
|
196
|
+
{
|
|
197
|
+
rule: 'LLM_RAG_NO_VALIDATION',
|
|
198
|
+
title: 'RAG Pipeline Without Input Validation',
|
|
199
|
+
regex: /(?:embed|embedding|vector|similarity_search|query)\s*\(\s*(?:req\.|request\.|body|query|params|input|user)/g,
|
|
200
|
+
severity: 'medium',
|
|
201
|
+
cwe: 'CWE-20',
|
|
202
|
+
owasp: 'LLM08',
|
|
203
|
+
description: 'User input passed directly to vector search/embedding without validation.',
|
|
204
|
+
fix: 'Validate and sanitize input before embedding. Limit query length.',
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
rule: 'LLM_RAG_NO_ACCESS_CONTROL',
|
|
208
|
+
title: 'RAG Without Access Control',
|
|
209
|
+
regex: /(?:pinecone|chroma|weaviate|qdrant|milvus).*(?:query|search|similarity)\s*\(/g,
|
|
210
|
+
severity: 'medium',
|
|
211
|
+
cwe: 'CWE-862',
|
|
212
|
+
owasp: 'LLM08',
|
|
213
|
+
confidence: 'low',
|
|
214
|
+
description: 'Vector database query without access control. Users may access other users\' data.',
|
|
215
|
+
fix: 'Add namespace/tenant filtering: filter by userId in vector DB queries',
|
|
216
|
+
},
|
|
217
|
+
|
|
218
|
+
// ── Prompt Injection Patterns (content-level detection) ────────────────────
|
|
219
|
+
{
|
|
220
|
+
rule: 'PROMPT_INJECTION_PATTERN',
|
|
221
|
+
title: 'Known Prompt Injection Pattern',
|
|
222
|
+
regex: /(?:ignore\s+(?:all\s+)?previous\s+instructions|disregard\s+(?:all\s+)?(?:previous|prior)|you\s+are\s+now\s+DAN|system\s*prompt|jailbreak|bypass\s+(?:your|the)\s+(?:rules|instructions|guidelines))/gi,
|
|
223
|
+
severity: 'high',
|
|
224
|
+
cwe: 'CWE-77',
|
|
225
|
+
owasp: 'LLM01',
|
|
226
|
+
description: 'Known prompt injection pattern detected in code. Ensure this is for testing only.',
|
|
227
|
+
fix: 'If in test data, add # ship-safe-ignore. If in user-facing code, add input filtering.',
|
|
228
|
+
},
|
|
229
|
+
];
|
|
230
|
+
|
|
231
|
+
export class LLMRedTeam extends BaseAgent {
|
|
232
|
+
constructor() {
|
|
233
|
+
super('LLMRedTeam', 'AI/LLM security audit based on OWASP LLM Top 10', 'llm');
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
async analyze(context) {
|
|
237
|
+
const { files } = context;
|
|
238
|
+
const codeFiles = files.filter(f => {
|
|
239
|
+
const ext = path.extname(f).toLowerCase();
|
|
240
|
+
return ['.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs', '.py', '.rb'].includes(ext);
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
let findings = [];
|
|
244
|
+
for (const file of codeFiles) {
|
|
245
|
+
findings = findings.concat(this.scanFileWithPatterns(file, PATTERNS));
|
|
246
|
+
}
|
|
247
|
+
return findings;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export default LLMRedTeam;
|