shield-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -0
- package/bin/shield-llm.js +2 -0
- package/dist/chunk-6HLD55BG.js +24282 -0
- package/dist/chunk-7D5WVZW5.js +574 -0
- package/dist/cli.js +1396 -0
- package/dist/index.js +24 -0
- package/dist/report-template.hbs +1726 -0
- package/dist/token-DULPX7LM.js +68 -0
- package/dist/token-util-HNUYLXRD.js +5 -0
- package/package.json +66 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,1396 @@
|
|
|
1
|
+
import { createRequire as __cjs_createRequire } from "module"; import { fileURLToPath as __cjs_fileURLToPath } from "url"; import { dirname as __cjs_dirname } from "path"; const __filename = __cjs_fileURLToPath(import.meta.url); const __dirname = __cjs_dirname(__filename); const require = __cjs_createRequire(import.meta.url);
|
|
2
|
+
import {
|
|
3
|
+
ConfigError,
|
|
4
|
+
ConfigSchema,
|
|
5
|
+
Scanner,
|
|
6
|
+
analyzeJudgeHealth,
|
|
7
|
+
buildTargetConfig,
|
|
8
|
+
calculateComplianceScore,
|
|
9
|
+
deserializeAttack,
|
|
10
|
+
deserializeMultiTurnAttack,
|
|
11
|
+
evaluateThresholds,
|
|
12
|
+
formatComplianceTerminal,
|
|
13
|
+
formatJson,
|
|
14
|
+
formatMarkdown,
|
|
15
|
+
formatSarif,
|
|
16
|
+
generatePdf,
|
|
17
|
+
loadConfig,
|
|
18
|
+
loadSystemPrompt
|
|
19
|
+
} from "./chunk-6HLD55BG.js";
|
|
20
|
+
import "./chunk-7D5WVZW5.js";
|
|
21
|
+
|
|
22
|
+
// src/cli.ts
|
|
23
|
+
import "dotenv/config";
|
|
24
|
+
import { Command, CommanderError } from "commander";
|
|
25
|
+
|
|
26
|
+
// src/credentials.ts
|
|
27
|
+
import {
|
|
28
|
+
readFileSync,
|
|
29
|
+
writeFileSync,
|
|
30
|
+
mkdirSync,
|
|
31
|
+
existsSync,
|
|
32
|
+
unlinkSync
|
|
33
|
+
} from "fs";
|
|
34
|
+
import { join } from "path";
|
|
35
|
+
import { homedir } from "os";
|
|
36
|
+
var DEFAULT_API_URL = "https://shield-llm.com";
|
|
37
|
+
function getConfigDir() {
|
|
38
|
+
const home = process.env.SHIELD_LLM_HOME ?? homedir();
|
|
39
|
+
return join(home, ".shield-llm");
|
|
40
|
+
}
|
|
41
|
+
function getCredentialsFile() {
|
|
42
|
+
return join(getConfigDir(), "credentials.json");
|
|
43
|
+
}
|
|
44
|
+
function loadCredentials() {
|
|
45
|
+
const file = getCredentialsFile();
|
|
46
|
+
if (!existsSync(file)) return null;
|
|
47
|
+
try {
|
|
48
|
+
const content = readFileSync(file, "utf-8");
|
|
49
|
+
const parsed = JSON.parse(content);
|
|
50
|
+
if (!parsed.apiKey || typeof parsed.apiKey !== "string") return null;
|
|
51
|
+
return {
|
|
52
|
+
apiKey: parsed.apiKey,
|
|
53
|
+
apiUrl: parsed.apiUrl ?? DEFAULT_API_URL
|
|
54
|
+
};
|
|
55
|
+
} catch {
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function resolveApiUrl(stored, override) {
|
|
60
|
+
const raw = override ?? process.env.SHIELD_API_URL ?? stored ?? DEFAULT_API_URL;
|
|
61
|
+
return raw.trim().replace(/\/+$/, "");
|
|
62
|
+
}
|
|
63
|
+
function saveCredentials(creds) {
|
|
64
|
+
const dir = getConfigDir();
|
|
65
|
+
if (!existsSync(dir)) {
|
|
66
|
+
mkdirSync(dir, { recursive: true });
|
|
67
|
+
}
|
|
68
|
+
const content = JSON.stringify(
|
|
69
|
+
{ apiKey: creds.apiKey, apiUrl: creds.apiUrl },
|
|
70
|
+
null,
|
|
71
|
+
2
|
|
72
|
+
);
|
|
73
|
+
writeFileSync(getCredentialsFile(), content, {
|
|
74
|
+
encoding: "utf-8",
|
|
75
|
+
mode: 384
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
function clearCredentials() {
|
|
79
|
+
const file = getCredentialsFile();
|
|
80
|
+
if (existsSync(file)) {
|
|
81
|
+
unlinkSync(file);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function getCredentialsPath() {
|
|
85
|
+
return getCredentialsFile();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// src/attacks-loader.ts
|
|
89
|
+
async function fetchAttacks(args) {
|
|
90
|
+
const params = new URLSearchParams({ preset: args.preset });
|
|
91
|
+
if (args.includeMultiTurn) params.set("includeMultiTurn", "true");
|
|
92
|
+
const url = `${args.apiUrl}/api/cli/attacks?${params.toString()}`;
|
|
93
|
+
let res;
|
|
94
|
+
try {
|
|
95
|
+
res = await fetch(url, {
|
|
96
|
+
headers: { Authorization: `Bearer ${args.apiKey}` }
|
|
97
|
+
});
|
|
98
|
+
} catch (err) {
|
|
99
|
+
const reason = err instanceof Error ? err.message : "Unknown error";
|
|
100
|
+
throw new Error(
|
|
101
|
+
`Could not reach the Shield LLM backend at ${args.apiUrl} (${reason}). The CLI needs a backend connection to fetch the attack catalogue.`
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
if (res.status === 401) {
|
|
105
|
+
throw new Error(
|
|
106
|
+
"API key is invalid or revoked. Run `shield-llm login` to re-authenticate."
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
if (res.status === 403) {
|
|
110
|
+
const body = await res.json().catch(() => ({}));
|
|
111
|
+
throw new Error(
|
|
112
|
+
body.error ?? "Plan does not allow this preset. Upgrade required."
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
if (!res.ok) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
`Backend returned HTTP ${res.status} fetching the attack catalogue.`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
try {
|
|
121
|
+
const body = await res.json();
|
|
122
|
+
return {
|
|
123
|
+
attacks: body.attacks.map(deserializeAttack),
|
|
124
|
+
multiTurnAttacks: (body.multiTurnAttacks ?? []).map(
|
|
125
|
+
deserializeMultiTurnAttack
|
|
126
|
+
),
|
|
127
|
+
source: "backend",
|
|
128
|
+
catalogVersion: body.catalogVersion
|
|
129
|
+
};
|
|
130
|
+
} catch (err) {
|
|
131
|
+
const reason = err instanceof Error ? err.message : "Unknown error";
|
|
132
|
+
throw new Error(
|
|
133
|
+
`Received a malformed attack catalogue from the backend (${reason}).`
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// src/lib/exit.ts
|
|
139
|
+
function setExitCode(code) {
|
|
140
|
+
process.exitCode = code;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// src/commands/attacks.ts
|
|
144
|
+
async function attacksCommand(opts) {
|
|
145
|
+
const stored = loadCredentials();
|
|
146
|
+
if (!stored) {
|
|
147
|
+
console.error("Not logged in. Run: shield-llm login --key <your-key>\n");
|
|
148
|
+
setExitCode(2);
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const apiUrl = resolveApiUrl(stored.apiUrl, opts.apiUrl);
|
|
152
|
+
let fetched;
|
|
153
|
+
try {
|
|
154
|
+
fetched = await fetchAttacks({
|
|
155
|
+
apiUrl,
|
|
156
|
+
apiKey: stored.apiKey,
|
|
157
|
+
preset: "full",
|
|
158
|
+
includeMultiTurn: true
|
|
159
|
+
});
|
|
160
|
+
} catch (err) {
|
|
161
|
+
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
162
|
+
console.error(`Failed to fetch attacks: ${msg}`);
|
|
163
|
+
setExitCode(3);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
const wanted = opts.severity?.toLowerCase();
|
|
167
|
+
const singleTurn = fetched.attacks.filter(
|
|
168
|
+
(a) => !wanted || a.severity.toLowerCase() === wanted
|
|
169
|
+
);
|
|
170
|
+
const multiTurn = fetched.multiTurnAttacks.filter(
|
|
171
|
+
(a) => !wanted || a.severity.toLowerCase() === wanted
|
|
172
|
+
);
|
|
173
|
+
const total = singleTurn.length + multiTurn.length;
|
|
174
|
+
console.log(
|
|
175
|
+
`
|
|
176
|
+
Available attacks (${total}) \u2014 catalog ${fetched.catalogVersion}:
|
|
177
|
+
`
|
|
178
|
+
);
|
|
179
|
+
for (const attack of [...singleTurn, ...multiTurn]) {
|
|
180
|
+
const flags = [];
|
|
181
|
+
if (attack.requiresMcpHarness) flags.push("mcp");
|
|
182
|
+
if (attack.requiresVisionTarget) flags.push("vision");
|
|
183
|
+
if (attack.requiresMultiAgent) flags.push("multi-agent");
|
|
184
|
+
const flagStr = flags.length ? ` [needs: ${flags.join(", ")}]` : "";
|
|
185
|
+
console.log(
|
|
186
|
+
` ${attack.id.padEnd(38)} ${attack.severity.padEnd(10)} ${attack.name}${flagStr}`
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
console.log("");
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// src/commands/init.ts
|
|
193
|
+
import { existsSync as existsSync2, writeFileSync as writeFileSync2 } from "fs";
|
|
194
|
+
import { resolve } from "path";
|
|
195
|
+
import { select, input, confirm } from "@inquirer/prompts";
|
|
196
|
+
var CONFIG_FILENAME = "shield.config.json";
|
|
197
|
+
async function initCommand() {
|
|
198
|
+
console.log("\nShield LLM \u2014 Configuration wizard\n");
|
|
199
|
+
if (!process.stdin.isTTY) {
|
|
200
|
+
console.error(
|
|
201
|
+
"`shield-llm init` is interactive and needs a terminal (TTY).\nRun it directly in your terminal, or write shield.config.json by hand\nand check it with `shield-llm validate`."
|
|
202
|
+
);
|
|
203
|
+
setExitCode(2);
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
const configPath = resolve(CONFIG_FILENAME);
|
|
207
|
+
if (existsSync2(configPath)) {
|
|
208
|
+
const overwrite = await confirm({
|
|
209
|
+
message: `${CONFIG_FILENAME} already exists. Overwrite?`,
|
|
210
|
+
default: false
|
|
211
|
+
});
|
|
212
|
+
if (!overwrite) {
|
|
213
|
+
console.log("Aborted.");
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
const url = await input({
|
|
218
|
+
message: "Endpoint URL of your chatbot:",
|
|
219
|
+
validate: (value) => {
|
|
220
|
+
try {
|
|
221
|
+
new URL(value);
|
|
222
|
+
return true;
|
|
223
|
+
} catch {
|
|
224
|
+
return "Please enter a valid URL (e.g. https://api.example.com/chat)";
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
});
|
|
228
|
+
const method = await select({
|
|
229
|
+
message: "HTTP method:",
|
|
230
|
+
choices: [
|
|
231
|
+
{ value: "POST", name: "POST (most chatbots)" },
|
|
232
|
+
{ value: "GET", name: "GET" },
|
|
233
|
+
{ value: "PUT", name: "PUT" }
|
|
234
|
+
],
|
|
235
|
+
default: "POST"
|
|
236
|
+
});
|
|
237
|
+
const auth = await collectAuth();
|
|
238
|
+
const { body, responseField } = await collectBodyShape();
|
|
239
|
+
const customHeaders = await collectCustomHeaders();
|
|
240
|
+
const preset = await select({
|
|
241
|
+
message: "Default scan preset:",
|
|
242
|
+
choices: [
|
|
243
|
+
{ value: "owasp", name: "owasp \u2014 OWASP LLM Top 10 (10 tests)" },
|
|
244
|
+
{ value: "quick", name: "quick \u2014 Find flaws fast (7 tests)" },
|
|
245
|
+
{ value: "standard", name: "standard \u2014 Recommended (39 tests)" },
|
|
246
|
+
{
|
|
247
|
+
value: "full",
|
|
248
|
+
name: "full \u2014 Comprehensive (all tests + crescendo)"
|
|
249
|
+
}
|
|
250
|
+
],
|
|
251
|
+
default: "standard"
|
|
252
|
+
});
|
|
253
|
+
const endpoint = {
|
|
254
|
+
url,
|
|
255
|
+
request: {
|
|
256
|
+
method,
|
|
257
|
+
...Object.keys(customHeaders).length > 0 && { headers: customHeaders },
|
|
258
|
+
body
|
|
259
|
+
},
|
|
260
|
+
response: { field: responseField }
|
|
261
|
+
};
|
|
262
|
+
if (auth.type !== "none") {
|
|
263
|
+
endpoint.auth = auth;
|
|
264
|
+
} else {
|
|
265
|
+
endpoint.auth = { type: "none" };
|
|
266
|
+
}
|
|
267
|
+
const config = { endpoint, preset };
|
|
268
|
+
const result = ConfigSchema.safeParse(config);
|
|
269
|
+
if (!result.success) {
|
|
270
|
+
const issues = result.error.issues.map((i) => ` - ${i.path.join(".")}: ${i.message}`).join("\n");
|
|
271
|
+
console.error(`
|
|
272
|
+
Generated config is invalid:
|
|
273
|
+
${issues}`);
|
|
274
|
+
console.error(
|
|
275
|
+
"This is a bug \u2014 please report it at https://github.com/anthropics/shield-llm/issues"
|
|
276
|
+
);
|
|
277
|
+
setExitCode(3);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
writeFileSync2(configPath, JSON.stringify(result.data, null, 2) + "\n");
|
|
281
|
+
console.log(`
|
|
282
|
+
\u2713 Config written to ${CONFIG_FILENAME}
|
|
283
|
+
`);
|
|
284
|
+
console.log(" Next steps:");
|
|
285
|
+
console.log(
|
|
286
|
+
" shield-llm login Authenticate (SaaS or License)"
|
|
287
|
+
);
|
|
288
|
+
console.log(" shield-llm validate Verify your configuration");
|
|
289
|
+
console.log(" shield-llm scan Run a scan with your preset");
|
|
290
|
+
console.log("");
|
|
291
|
+
}
|
|
292
|
+
async function collectAuth() {
|
|
293
|
+
const authType = await select({
|
|
294
|
+
message: "How does your endpoint authenticate?",
|
|
295
|
+
choices: [
|
|
296
|
+
{ value: "none", name: "No authentication" },
|
|
297
|
+
{ value: "bearer", name: "Bearer token" },
|
|
298
|
+
{ value: "api-key", name: "API key in header" },
|
|
299
|
+
{ value: "oauth2", name: "OAuth2 client credentials" }
|
|
300
|
+
]
|
|
301
|
+
});
|
|
302
|
+
if (authType === "none") return { type: "none" };
|
|
303
|
+
if (authType === "bearer") {
|
|
304
|
+
const token = await input({
|
|
305
|
+
message: "Bearer token (use $ENV_VAR to read from env):",
|
|
306
|
+
validate: (v) => v.length >= 1 ? true : "Token is required"
|
|
307
|
+
});
|
|
308
|
+
warnIfHardcodedSecret(token);
|
|
309
|
+
return { type: "bearer", token };
|
|
310
|
+
}
|
|
311
|
+
if (authType === "api-key") {
|
|
312
|
+
const header = await input({
|
|
313
|
+
message: "Header name:",
|
|
314
|
+
default: "X-API-Key"
|
|
315
|
+
});
|
|
316
|
+
const value = await input({
|
|
317
|
+
message: "API key value (use $ENV_VAR to read from env):",
|
|
318
|
+
validate: (v) => v.length >= 1 ? true : "Value is required"
|
|
319
|
+
});
|
|
320
|
+
warnIfHardcodedSecret(value);
|
|
321
|
+
return { type: "api-key", header, value };
|
|
322
|
+
}
|
|
323
|
+
const clientId = await input({
|
|
324
|
+
message: "Client ID (use $ENV_VAR to read from env):"
|
|
325
|
+
});
|
|
326
|
+
const clientSecret = await input({
|
|
327
|
+
message: "Client secret (use $ENV_VAR to read from env):"
|
|
328
|
+
});
|
|
329
|
+
warnIfHardcodedSecret(clientSecret);
|
|
330
|
+
const tokenUrl = await input({
|
|
331
|
+
message: "Token URL:",
|
|
332
|
+
validate: (v) => {
|
|
333
|
+
try {
|
|
334
|
+
new URL(v);
|
|
335
|
+
return true;
|
|
336
|
+
} catch {
|
|
337
|
+
return "Must be a valid URL";
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
return { type: "oauth2", clientId, clientSecret, tokenUrl };
|
|
342
|
+
}
|
|
343
|
+
async function collectBodyShape() {
|
|
344
|
+
const shape = await select({
|
|
345
|
+
message: "Request body shape:",
|
|
346
|
+
choices: [
|
|
347
|
+
{
|
|
348
|
+
value: "wrapper",
|
|
349
|
+
name: "JSON wrapper (most chatbots \u2014 { message: '{{prompt}}', ...fixed }) "
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
value: "literal",
|
|
353
|
+
name: "Literal prompt-only ({ prompt: '{{prompt}}' })"
|
|
354
|
+
}
|
|
355
|
+
],
|
|
356
|
+
default: "wrapper"
|
|
357
|
+
});
|
|
358
|
+
if (shape === "literal") {
|
|
359
|
+
const responseField2 = await input({
|
|
360
|
+
message: "Response field (dot-notation):",
|
|
361
|
+
default: "response"
|
|
362
|
+
});
|
|
363
|
+
return { body: { prompt: "{{prompt}}" }, responseField: responseField2 };
|
|
364
|
+
}
|
|
365
|
+
const promptField = await input({
|
|
366
|
+
message: "Field name that carries the user prompt:",
|
|
367
|
+
default: "message"
|
|
368
|
+
});
|
|
369
|
+
const body = { [promptField]: "{{prompt}}" };
|
|
370
|
+
const addExtra = await confirm({
|
|
371
|
+
message: "Add extra fixed fields to the request body (assistant_id, thread_id, ...) ?",
|
|
372
|
+
default: false
|
|
373
|
+
});
|
|
374
|
+
if (addExtra) {
|
|
375
|
+
let more = true;
|
|
376
|
+
while (more) {
|
|
377
|
+
const key = await input({
|
|
378
|
+
message: "Field name:",
|
|
379
|
+
validate: (v) => v.length >= 1 ? true : "Required"
|
|
380
|
+
});
|
|
381
|
+
const value = await input({
|
|
382
|
+
message: `Value for "${key}":`
|
|
383
|
+
});
|
|
384
|
+
body[key] = value;
|
|
385
|
+
more = await confirm({ message: "Another field?", default: false });
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
const responseField = await input({
|
|
389
|
+
message: "Response field path (e.g. response, data.reply, choices[0].message.content):",
|
|
390
|
+
default: "response"
|
|
391
|
+
});
|
|
392
|
+
return { body, responseField };
|
|
393
|
+
}
|
|
394
|
+
async function collectCustomHeaders() {
|
|
395
|
+
const add = await confirm({
|
|
396
|
+
message: "Add custom request headers?",
|
|
397
|
+
default: false
|
|
398
|
+
});
|
|
399
|
+
if (!add) return {};
|
|
400
|
+
const headers = {};
|
|
401
|
+
let more = true;
|
|
402
|
+
while (more) {
|
|
403
|
+
const name = await input({
|
|
404
|
+
message: "Header name:",
|
|
405
|
+
validate: (v) => v.length >= 1 ? true : "Required"
|
|
406
|
+
});
|
|
407
|
+
const value = await input({
|
|
408
|
+
message: `Value for "${name}" (use $ENV_VAR to read from env):`
|
|
409
|
+
});
|
|
410
|
+
warnIfHardcodedSecret(value);
|
|
411
|
+
headers[name] = value;
|
|
412
|
+
more = await confirm({ message: "Another header?", default: false });
|
|
413
|
+
}
|
|
414
|
+
return headers;
|
|
415
|
+
}
|
|
416
|
+
function warnIfHardcodedSecret(value) {
|
|
417
|
+
if (value && !value.startsWith("$")) {
|
|
418
|
+
console.warn(
|
|
419
|
+
"\n Tip: prefer $ENV_VAR (e.g. $BEARER_TOKEN) so secrets stay out of shield.config.json.\n"
|
|
420
|
+
);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// src/commands/login.ts
|
|
425
|
+
async function loginCommand(opts) {
|
|
426
|
+
const key = opts.key;
|
|
427
|
+
if (!key.startsWith("sk_shield_")) {
|
|
428
|
+
console.error('Invalid key format. API keys start with "sk_shield_".');
|
|
429
|
+
setExitCode(2);
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
const apiUrl = resolveApiUrl(void 0, opts.apiUrl);
|
|
433
|
+
saveCredentials({ apiKey: key, apiUrl });
|
|
434
|
+
console.log(`Credentials saved to ${getCredentialsPath()}`);
|
|
435
|
+
try {
|
|
436
|
+
const res = await fetch(`${apiUrl}/api/cli/scans`, {
|
|
437
|
+
method: "POST",
|
|
438
|
+
headers: {
|
|
439
|
+
"Content-Type": "application/json",
|
|
440
|
+
Authorization: `Bearer ${key}`
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
if (res.status === 401) {
|
|
444
|
+
console.warn(
|
|
445
|
+
"Warning: API key appears to be invalid. Check your key and try again."
|
|
446
|
+
);
|
|
447
|
+
} else if (res.ok) {
|
|
448
|
+
const data = await res.json();
|
|
449
|
+
console.log(
|
|
450
|
+
`Authenticated to ${apiUrl}! Plan: ${data.plan} (${data.remaining}/${data.limit} scans remaining)`
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
} catch {
|
|
454
|
+
console.log(
|
|
455
|
+
"Credentials saved. Could not verify \u2014 the backend may be unreachable."
|
|
456
|
+
);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// src/commands/logout.ts
|
|
461
|
+
function logoutCommand() {
|
|
462
|
+
clearCredentials();
|
|
463
|
+
console.log("Credentials removed.");
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// src/commands/presets.ts
|
|
467
|
+
async function presetsCommand(opts) {
|
|
468
|
+
const stored = loadCredentials();
|
|
469
|
+
if (!stored) {
|
|
470
|
+
console.error("Not logged in. Run: shield-llm login --key <your-key>\n");
|
|
471
|
+
setExitCode(2);
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
const apiUrl = resolveApiUrl(stored.apiUrl, opts.apiUrl);
|
|
475
|
+
let presets;
|
|
476
|
+
try {
|
|
477
|
+
const res = await fetch(`${apiUrl}/api/cli/presets`, {
|
|
478
|
+
headers: { Authorization: `Bearer ${stored.apiKey}` }
|
|
479
|
+
});
|
|
480
|
+
if (res.status === 401) {
|
|
481
|
+
console.error(
|
|
482
|
+
"API key is invalid or revoked. Run `shield-llm login` to re-authenticate."
|
|
483
|
+
);
|
|
484
|
+
setExitCode(2);
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
if (!res.ok) {
|
|
488
|
+
const body = await res.json().catch(() => ({}));
|
|
489
|
+
console.error(
|
|
490
|
+
`Failed to fetch presets: ${body.error ?? `HTTP ${res.status}`}`
|
|
491
|
+
);
|
|
492
|
+
setExitCode(3);
|
|
493
|
+
return;
|
|
494
|
+
}
|
|
495
|
+
({ presets } = await res.json());
|
|
496
|
+
} catch (err) {
|
|
497
|
+
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
498
|
+
console.error(`Failed to fetch presets: ${msg}`);
|
|
499
|
+
setExitCode(3);
|
|
500
|
+
return;
|
|
501
|
+
}
|
|
502
|
+
console.log("\nAvailable presets:\n");
|
|
503
|
+
for (const preset of presets) {
|
|
504
|
+
const lock = preset.allowed ? "" : " (upgrade required)";
|
|
505
|
+
console.log(
|
|
506
|
+
` ${preset.id.padEnd(12)} ${preset.name.padEnd(20)} ${preset.description}${lock}`
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
console.log("");
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// src/commands/report.ts
|
|
513
|
+
import { writeFileSync as writeFileSync3 } from "fs";
|
|
514
|
+
import { resolve as resolve2 } from "path";
|
|
515
|
+
function renderScanMarkdown(scan) {
|
|
516
|
+
const detected = scan.vulns.filter((v) => v.status === "DETECTED");
|
|
517
|
+
const passed = scan.vulns.length - detected.length;
|
|
518
|
+
const lines = [
|
|
519
|
+
`# Shield LLM scan ${scan.id}`,
|
|
520
|
+
"",
|
|
521
|
+
`- Target: \`${scan.targetUrl}\``,
|
|
522
|
+
`- Score: **${scan.score}** (${scan.grade})`,
|
|
523
|
+
`- Date: ${scan.createdAt}`,
|
|
524
|
+
`- Tests: ${scan.testsRun}/${scan.testsTotal} run, ${detected.length} vulnerable, ${passed} secure`,
|
|
525
|
+
"",
|
|
526
|
+
"## Vulnerabilities detected",
|
|
527
|
+
""
|
|
528
|
+
];
|
|
529
|
+
if (detected.length === 0) {
|
|
530
|
+
lines.push("_None_");
|
|
531
|
+
} else {
|
|
532
|
+
lines.push("| Severity | Name | Notes |");
|
|
533
|
+
lines.push("|---|---|---|");
|
|
534
|
+
for (const v of detected) {
|
|
535
|
+
lines.push(
|
|
536
|
+
`| ${v.severity} | ${v.name} | ${(v.description ?? "").slice(0, 80)} |`
|
|
537
|
+
);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
return lines.join("\n") + "\n";
|
|
541
|
+
}
|
|
542
|
+
async function reportCommand(scanId, opts) {
|
|
543
|
+
const stored = loadCredentials();
|
|
544
|
+
if (!stored) {
|
|
545
|
+
console.error("Not logged in. Run `shield-llm login` first.");
|
|
546
|
+
process.exitCode = 2;
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
const apiUrl = resolveApiUrl(stored.apiUrl, opts.apiUrl);
|
|
550
|
+
let res;
|
|
551
|
+
try {
|
|
552
|
+
res = await fetch(
|
|
553
|
+
`${apiUrl}/api/cli/reports/${encodeURIComponent(scanId)}`,
|
|
554
|
+
{
|
|
555
|
+
headers: { Authorization: `Bearer ${stored.apiKey}` }
|
|
556
|
+
}
|
|
557
|
+
);
|
|
558
|
+
} catch (err) {
|
|
559
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
560
|
+
console.error(`Failed to reach backend: ${msg}`);
|
|
561
|
+
process.exitCode = 3;
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
if (res.status === 401) {
|
|
565
|
+
await res.text().catch(() => void 0);
|
|
566
|
+
console.error(
|
|
567
|
+
"API key invalid. Run `shield-llm login` to re-authenticate."
|
|
568
|
+
);
|
|
569
|
+
process.exitCode = 2;
|
|
570
|
+
return;
|
|
571
|
+
}
|
|
572
|
+
if (res.status === 404) {
|
|
573
|
+
await res.text().catch(() => void 0);
|
|
574
|
+
console.error(`Scan "${scanId}" not found.`);
|
|
575
|
+
process.exitCode = 2;
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
578
|
+
if (!res.ok) {
|
|
579
|
+
await res.text().catch(() => void 0);
|
|
580
|
+
console.error(`Backend returned HTTP ${res.status}`);
|
|
581
|
+
process.exitCode = 3;
|
|
582
|
+
return;
|
|
583
|
+
}
|
|
584
|
+
const scan = await res.json();
|
|
585
|
+
let content;
|
|
586
|
+
if (opts.output === "markdown") {
|
|
587
|
+
content = renderScanMarkdown(scan);
|
|
588
|
+
} else {
|
|
589
|
+
content = JSON.stringify(scan, null, 2);
|
|
590
|
+
}
|
|
591
|
+
if (opts.outputFile) {
|
|
592
|
+
writeFileSync3(resolve2(opts.outputFile), content, "utf-8");
|
|
593
|
+
console.error(`Report written to ${opts.outputFile}`);
|
|
594
|
+
} else {
|
|
595
|
+
console.log(content);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// src/commands/scan.ts
|
|
600
|
+
import { writeFileSync as writeFileSync4 } from "fs";
|
|
601
|
+
import { resolve as resolve3 } from "path";
|
|
602
|
+
|
|
603
|
+
// src/custom-tests-loader.ts
|
|
604
|
+
async function fetchCustomTests(apiUrl, apiKey) {
|
|
605
|
+
const res = await fetch(`${apiUrl}/api/cli/tests`, {
|
|
606
|
+
headers: {
|
|
607
|
+
Authorization: `Bearer ${apiKey}`
|
|
608
|
+
}
|
|
609
|
+
});
|
|
610
|
+
if (res.status === 401) {
|
|
611
|
+
throw new Error(
|
|
612
|
+
"API key is invalid or revoked. Run `shield-llm login` to re-authenticate."
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
if (!res.ok) {
|
|
616
|
+
throw new Error(`Failed to fetch custom tests: HTTP ${res.status}`);
|
|
617
|
+
}
|
|
618
|
+
const scans = await res.json();
|
|
619
|
+
return scans.filter((scan) => scan.isActive).flatMap(
|
|
620
|
+
(scan) => scan.tests.map((test) => ({
|
|
621
|
+
name: test.name,
|
|
622
|
+
prompt: test.prompt,
|
|
623
|
+
acceptanceCriteria: test.acceptanceCriteria
|
|
624
|
+
}))
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// src/evaluators/backend-evaluator.ts
|
|
629
|
+
var MAX_RESPONSE_CHARS = 19500;
|
|
630
|
+
var HEAD_CHARS = 15e3;
|
|
631
|
+
var TAIL_CHARS = 4e3;
|
|
632
|
+
function truncateResponse(text) {
|
|
633
|
+
if (text.length <= MAX_RESPONSE_CHARS) return text;
|
|
634
|
+
const cut = text.length - HEAD_CHARS - TAIL_CHARS;
|
|
635
|
+
return `${text.slice(0, HEAD_CHARS)}
|
|
636
|
+
|
|
637
|
+
[...TRUNCATED ${cut} chars...]
|
|
638
|
+
|
|
639
|
+
${text.slice(-TAIL_CHARS)}`;
|
|
640
|
+
}
|
|
641
|
+
function truncateInput(input2) {
|
|
642
|
+
return {
|
|
643
|
+
...input2,
|
|
644
|
+
response: truncateResponse(input2.response),
|
|
645
|
+
turns: input2.turns?.map((t) => ({
|
|
646
|
+
prompt: t.prompt,
|
|
647
|
+
response: truncateResponse(t.response)
|
|
648
|
+
}))
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
var BackendEvaluator = class {
|
|
652
|
+
constructor(apiUrl, apiKey) {
|
|
653
|
+
this.apiUrl = apiUrl;
|
|
654
|
+
this.apiKey = apiKey;
|
|
655
|
+
}
|
|
656
|
+
async evaluate(input2) {
|
|
657
|
+
const response = await fetch(`${this.apiUrl}/api/cli/evaluate`, {
|
|
658
|
+
method: "POST",
|
|
659
|
+
headers: {
|
|
660
|
+
"Content-Type": "application/json",
|
|
661
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
662
|
+
},
|
|
663
|
+
body: JSON.stringify(truncateInput(input2))
|
|
664
|
+
});
|
|
665
|
+
if (response.status === 401) {
|
|
666
|
+
throw new Error(
|
|
667
|
+
"Invalid API key. Run `shield-llm login` to re-authenticate."
|
|
668
|
+
);
|
|
669
|
+
}
|
|
670
|
+
if (response.status === 403) {
|
|
671
|
+
const body = await response.json().catch(() => ({}));
|
|
672
|
+
throw new Error(
|
|
673
|
+
body.error ?? `Scan quota exceeded. Upgrade your plan at ${DEFAULT_API_URL}/settings`
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
if (response.status === 429) {
|
|
677
|
+
throw new Error("Rate limit exceeded. Please wait and try again.");
|
|
678
|
+
}
|
|
679
|
+
if (!response.ok) {
|
|
680
|
+
const body = await response.text().catch(() => "");
|
|
681
|
+
let userMessage = `Backend evaluation failed (HTTP ${response.status}). Please try again.`;
|
|
682
|
+
try {
|
|
683
|
+
const parsed = JSON.parse(body);
|
|
684
|
+
if (typeof parsed.error === "string" && parsed.error.length > 0) {
|
|
685
|
+
userMessage = `Backend evaluation failed (${response.status}): ${parsed.error}`;
|
|
686
|
+
}
|
|
687
|
+
} catch {
|
|
688
|
+
if (body) {
|
|
689
|
+
console.error(
|
|
690
|
+
`[BackendEvaluator] Non-JSON ${response.status} response body (first 500 chars): ${body.slice(0, 500)}`
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
throw new Error(userMessage);
|
|
695
|
+
}
|
|
696
|
+
return response.json();
|
|
697
|
+
}
|
|
698
|
+
};
|
|
699
|
+
|
|
700
|
+
// src/display/progress.ts
|
|
701
|
+
import chalk from "chalk";
|
|
702
|
+
import ora from "ora";
|
|
703
|
+
var spinner = ora();
|
|
704
|
+
var currentAttackName = "";
|
|
705
|
+
function handleEvent(event) {
|
|
706
|
+
switch (event.type) {
|
|
707
|
+
case "scan:start":
|
|
708
|
+
spinner.start(
|
|
709
|
+
chalk.cyan(`Starting scan (${event.totalAttacks} attacks)...`)
|
|
710
|
+
);
|
|
711
|
+
break;
|
|
712
|
+
case "attack:start":
|
|
713
|
+
currentAttackName = event.attack.name;
|
|
714
|
+
spinner.text = chalk.cyan(
|
|
715
|
+
`[${event.index + 1}/${event.total}] ${event.attack.name} (${severityColor(event.attack.severity)})`
|
|
716
|
+
);
|
|
717
|
+
break;
|
|
718
|
+
case "attack:complete": {
|
|
719
|
+
const { result } = event;
|
|
720
|
+
const prefix = `[${event.index + 1}/${event.total}]`;
|
|
721
|
+
if (result.error) {
|
|
722
|
+
spinner.warn(
|
|
723
|
+
`${prefix} ${result.name} \u2014 ${chalk.yellow("error")}: ${result.error}`
|
|
724
|
+
);
|
|
725
|
+
} else if (result.vulnerable) {
|
|
726
|
+
spinner.fail(
|
|
727
|
+
`${prefix} ${result.name} \u2014 ${chalk.red("VULNERABLE")} (${severityColor(result.severity)})`
|
|
728
|
+
);
|
|
729
|
+
} else {
|
|
730
|
+
spinner.succeed(`${prefix} ${result.name} \u2014 ${chalk.green("secure")}`);
|
|
731
|
+
}
|
|
732
|
+
if (event.index < event.total - 1) {
|
|
733
|
+
spinner.start();
|
|
734
|
+
}
|
|
735
|
+
break;
|
|
736
|
+
}
|
|
737
|
+
case "crescendo:turn":
|
|
738
|
+
spinner.text = chalk.cyan(
|
|
739
|
+
`${currentAttackName} \u2014 turn ${event.turn}/${event.totalTurns}`
|
|
740
|
+
);
|
|
741
|
+
break;
|
|
742
|
+
case "scan:rate_limited":
|
|
743
|
+
spinner.fail(
|
|
744
|
+
chalk.red(
|
|
745
|
+
`Rate limited by target \u2014 skipping remaining attacks${event.retryAfterSec ? ` (retry after ${event.retryAfterSec}s)` : ""}`
|
|
746
|
+
)
|
|
747
|
+
);
|
|
748
|
+
spinner.start();
|
|
749
|
+
break;
|
|
750
|
+
case "scan:complete":
|
|
751
|
+
spinner.stop();
|
|
752
|
+
break;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
function printSummary(report, thresholds) {
|
|
756
|
+
console.log("");
|
|
757
|
+
console.log(chalk.bold("\u2550\u2550\u2550 Shield LLM Scan Results \u2550\u2550\u2550"));
|
|
758
|
+
console.log("");
|
|
759
|
+
console.log(` Target: ${chalk.white(report.target)}`);
|
|
760
|
+
console.log(` Score: ${scoreColor(report.score)}`);
|
|
761
|
+
console.log(` Grade: ${gradeColor(report.grade)}`);
|
|
762
|
+
console.log(` Duration: ${(report.durationMs / 1e3).toFixed(1)}s`);
|
|
763
|
+
console.log("");
|
|
764
|
+
const vulnerable = report.attacks.filter((a) => a.vulnerable);
|
|
765
|
+
const secure = report.attacks.filter((a) => !a.vulnerable && !a.error);
|
|
766
|
+
const errors = report.attacks.filter((a) => a.error);
|
|
767
|
+
console.log(
|
|
768
|
+
` ${chalk.red(`${vulnerable.length} vulnerable`)} \xB7 ${chalk.green(`${secure.length} secure`)} \xB7 ${chalk.yellow(`${errors.length} errors`)}`
|
|
769
|
+
);
|
|
770
|
+
if (vulnerable.length > 0) {
|
|
771
|
+
console.log("");
|
|
772
|
+
console.log(chalk.red.bold(" Vulnerabilities:"));
|
|
773
|
+
for (const a of vulnerable) {
|
|
774
|
+
const conf = a.llmEvaluation ? ` (${(a.llmEvaluation.confidence * 100).toFixed(0)}% confidence)` : "";
|
|
775
|
+
console.log(
|
|
776
|
+
` ${chalk.red("\u2717")} ${a.name} \u2014 ${severityColor(a.severity)}${conf}`
|
|
777
|
+
);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
const judge = analyzeJudgeHealth(report);
|
|
781
|
+
if (judge.degraded) {
|
|
782
|
+
console.log("");
|
|
783
|
+
console.log(
|
|
784
|
+
chalk.bgYellow.black.bold(" JUDGE DEGRADED ") + chalk.yellow(
|
|
785
|
+
` ${judge.degradedCount}/${judge.totalEvaluated} verdicts fell back to regex (${judge.errorSummary}).`
|
|
786
|
+
)
|
|
787
|
+
);
|
|
788
|
+
console.log(
|
|
789
|
+
chalk.yellow(
|
|
790
|
+
" Regex-only verdicts are less reliable than LLM-assisted ones. False positives likely. Fix the LLM provider config and re-run."
|
|
791
|
+
)
|
|
792
|
+
);
|
|
793
|
+
}
|
|
794
|
+
console.log("");
|
|
795
|
+
if (thresholds.passed) {
|
|
796
|
+
console.log(chalk.green.bold(" \u2713 PASSED \u2014 all thresholds met"));
|
|
797
|
+
} else {
|
|
798
|
+
console.log(chalk.red.bold(" \u2717 FAILED \u2014 threshold violations:"));
|
|
799
|
+
for (const reason of thresholds.reasons) {
|
|
800
|
+
console.log(` - ${reason}`);
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
console.log("");
|
|
804
|
+
}
|
|
805
|
+
function severityColor(severity) {
|
|
806
|
+
switch (severity) {
|
|
807
|
+
case "CRITICAL":
|
|
808
|
+
return chalk.red.bold(severity);
|
|
809
|
+
case "HIGH":
|
|
810
|
+
case "CRESCENDO":
|
|
811
|
+
case "COMBO":
|
|
812
|
+
return chalk.yellow(severity);
|
|
813
|
+
case "MEDIUM":
|
|
814
|
+
return chalk.blue(severity);
|
|
815
|
+
default:
|
|
816
|
+
return chalk.gray(severity);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
function scoreColor(score) {
|
|
820
|
+
if (score >= 90) return chalk.green.bold(`${score}/100`);
|
|
821
|
+
if (score >= 75) return chalk.green(`${score}/100`);
|
|
822
|
+
if (score >= 60) return chalk.yellow(`${score}/100`);
|
|
823
|
+
if (score >= 40) return chalk.red(`${score}/100`);
|
|
824
|
+
return chalk.red.bold(`${score}/100`);
|
|
825
|
+
}
|
|
826
|
+
function gradeColor(grade) {
|
|
827
|
+
switch (grade) {
|
|
828
|
+
case "A":
|
|
829
|
+
return chalk.green.bold(grade);
|
|
830
|
+
case "B":
|
|
831
|
+
return chalk.green(grade);
|
|
832
|
+
case "C":
|
|
833
|
+
return chalk.yellow(grade);
|
|
834
|
+
case "D":
|
|
835
|
+
return chalk.red(grade);
|
|
836
|
+
default:
|
|
837
|
+
return chalk.red.bold(grade);
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
// src/lib/sanitize.ts
|
|
842
|
+
var DEFAULTS = {
|
|
843
|
+
responseMaxChars: 500,
|
|
844
|
+
reasoningMaxChars: 500,
|
|
845
|
+
evidenceItemMaxChars: 200,
|
|
846
|
+
evidenceMaxItems: 5,
|
|
847
|
+
errorMaxChars: 200
|
|
848
|
+
};
|
|
849
|
+
function truncateString(value, max) {
|
|
850
|
+
if (value.length <= max) return value;
|
|
851
|
+
return value.slice(0, max - 1) + "\u2026";
|
|
852
|
+
}
|
|
853
|
+
function sanitizeReportForDisk(report, opts = {}) {
|
|
854
|
+
const cfg = { ...DEFAULTS, ...opts };
|
|
855
|
+
return {
|
|
856
|
+
...report,
|
|
857
|
+
attacks: report.attacks.map((attack) => {
|
|
858
|
+
const sanitized = { ...attack };
|
|
859
|
+
if (typeof attack.response === "string") {
|
|
860
|
+
sanitized.response = truncateString(
|
|
861
|
+
attack.response,
|
|
862
|
+
cfg.responseMaxChars
|
|
863
|
+
);
|
|
864
|
+
}
|
|
865
|
+
if (attack.error) {
|
|
866
|
+
sanitized.error = truncateString(attack.error, cfg.errorMaxChars);
|
|
867
|
+
}
|
|
868
|
+
if (attack.llmEvaluation) {
|
|
869
|
+
const ev = attack.llmEvaluation;
|
|
870
|
+
sanitized.llmEvaluation = {
|
|
871
|
+
...ev,
|
|
872
|
+
reasoning: ev.reasoning ? truncateString(ev.reasoning, cfg.reasoningMaxChars) : ev.reasoning,
|
|
873
|
+
evidence: ev.evidence ? ev.evidence.slice(0, cfg.evidenceMaxItems).map((e) => truncateString(e, cfg.evidenceItemMaxChars)) : ev.evidence
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
return sanitized;
|
|
877
|
+
})
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/lib/format.ts
|
|
882
|
+
function truncate(str, max) {
|
|
883
|
+
const oneLine = str.replace(/\n/g, " ").trim();
|
|
884
|
+
return oneLine.length <= max ? oneLine : oneLine.slice(0, max - 3) + "...";
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// src/commands/scan.ts
|
|
888
|
+
function buildAuthFromFlags(opts) {
|
|
889
|
+
if (!opts.auth) return void 0;
|
|
890
|
+
switch (opts.auth) {
|
|
891
|
+
case "bearer":
|
|
892
|
+
if (!opts.token)
|
|
893
|
+
throw new ConfigError("--token is required with --auth bearer");
|
|
894
|
+
return { type: "bearer", token: opts.token };
|
|
895
|
+
case "api-key":
|
|
896
|
+
if (!opts.token)
|
|
897
|
+
throw new ConfigError("--token is required with --auth api-key");
|
|
898
|
+
return {
|
|
899
|
+
type: "api-key",
|
|
900
|
+
header: opts.authHeader ?? "X-API-Key",
|
|
901
|
+
value: opts.token
|
|
902
|
+
};
|
|
903
|
+
// oauth2 requires too many params for CLI flags — use config file instead
|
|
904
|
+
default:
|
|
905
|
+
throw new ConfigError(
|
|
906
|
+
`Unknown or unsupported auth type for CLI flags: "${opts.auth}". OAuth2 must be configured via shield.config.json.`
|
|
907
|
+
);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
async function uploadReportToBackend(report, credentials, scanId) {
|
|
911
|
+
const vulnerabilities = report.attacks.filter((a) => !a.error && !a.rateLimited).map((a) => ({
|
|
912
|
+
// attackId lets the backend validate against its known catalogue + the
|
|
913
|
+
// user's custom tests, rejecting fabricated vulnerability rows.
|
|
914
|
+
attackId: a.id,
|
|
915
|
+
name: a.name,
|
|
916
|
+
severity: a.severity,
|
|
917
|
+
vulnerabilityType: a.id,
|
|
918
|
+
vulnerable: a.vulnerable,
|
|
919
|
+
description: a.name,
|
|
920
|
+
evidence: a.prompt && a.response ? JSON.stringify({
|
|
921
|
+
prompt: a.prompt,
|
|
922
|
+
response: truncate(a.response, 2e3)
|
|
923
|
+
}) : a.response ? JSON.stringify({
|
|
924
|
+
prompt: "",
|
|
925
|
+
response: truncate(a.response, 2e3)
|
|
926
|
+
}) : void 0,
|
|
927
|
+
status: a.vulnerable ? "DETECTED" : "PASSED",
|
|
928
|
+
evaluationMethod: a.llmEvaluation?.evaluationMethod,
|
|
929
|
+
confidence: a.llmEvaluation?.confidence,
|
|
930
|
+
reasoning: a.llmEvaluation?.reasoning,
|
|
931
|
+
captureStatus: a.captureStatus ?? "captured"
|
|
932
|
+
}));
|
|
933
|
+
const testsRun = report.attacks.filter(
|
|
934
|
+
(a) => !a.error && !a.rateLimited
|
|
935
|
+
).length;
|
|
936
|
+
const payload = {
|
|
937
|
+
scanId,
|
|
938
|
+
targetUrl: report.target,
|
|
939
|
+
chatbotType: "api",
|
|
940
|
+
summary: `CLI scan \u2014 ${report.config.preset} preset \u2014 ${testsRun}/${report.config.totalAttacks} tests`,
|
|
941
|
+
testsRun,
|
|
942
|
+
testsTotal: report.config.totalAttacks,
|
|
943
|
+
durationMs: report.durationMs,
|
|
944
|
+
vulnerabilities
|
|
945
|
+
};
|
|
946
|
+
const res = await fetch(`${credentials.apiUrl}/api/cli/reports`, {
|
|
947
|
+
method: "POST",
|
|
948
|
+
headers: {
|
|
949
|
+
"Content-Type": "application/json",
|
|
950
|
+
Authorization: `Bearer ${credentials.apiKey}`
|
|
951
|
+
},
|
|
952
|
+
body: JSON.stringify(payload)
|
|
953
|
+
});
|
|
954
|
+
if (!res.ok) {
|
|
955
|
+
const body = await res.text().catch(() => "");
|
|
956
|
+
throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
async function scanCommand(opts) {
|
|
960
|
+
try {
|
|
961
|
+
if (opts.ci) {
|
|
962
|
+
opts.output = opts.output ?? "json";
|
|
963
|
+
opts.progress = false;
|
|
964
|
+
}
|
|
965
|
+
const info = (msg) => {
|
|
966
|
+
if (opts.ci) console.error(msg);
|
|
967
|
+
else console.log(msg);
|
|
968
|
+
};
|
|
969
|
+
const stored = loadCredentials();
|
|
970
|
+
const credentials = stored ? { ...stored, apiUrl: resolveApiUrl(stored.apiUrl, opts.apiUrl) } : null;
|
|
971
|
+
if (!credentials) {
|
|
972
|
+
console.error(
|
|
973
|
+
`Not logged in. Run \`shield-llm login\` to authenticate.
|
|
974
|
+
SaaS: shield-llm login
|
|
975
|
+
License: shield-llm login --key <license-key> --api-url https://your-instance
|
|
976
|
+
`
|
|
977
|
+
);
|
|
978
|
+
setExitCode(2);
|
|
979
|
+
return;
|
|
980
|
+
}
|
|
981
|
+
let reservedScanId = null;
|
|
982
|
+
try {
|
|
983
|
+
const res = await fetch(`${credentials.apiUrl}/api/cli/scans`, {
|
|
984
|
+
method: "POST",
|
|
985
|
+
headers: {
|
|
986
|
+
"Content-Type": "application/json",
|
|
987
|
+
Authorization: `Bearer ${credentials.apiKey}`
|
|
988
|
+
}
|
|
989
|
+
});
|
|
990
|
+
if (res.status === 401) {
|
|
991
|
+
console.error(
|
|
992
|
+
"API key is invalid or revoked. Run `shield-llm login` to re-authenticate."
|
|
993
|
+
);
|
|
994
|
+
setExitCode(2);
|
|
995
|
+
return;
|
|
996
|
+
}
|
|
997
|
+
if (res.status === 403) {
|
|
998
|
+
const body = await res.json().catch(() => ({}));
|
|
999
|
+
console.error(
|
|
1000
|
+
body.error ?? `Monthly scan limit reached. Upgrade at ${DEFAULT_API_URL}/settings`
|
|
1001
|
+
);
|
|
1002
|
+
setExitCode(2);
|
|
1003
|
+
return;
|
|
1004
|
+
}
|
|
1005
|
+
if (!res.ok) {
|
|
1006
|
+
console.error(`Failed to start scan: HTTP ${res.status}`);
|
|
1007
|
+
setExitCode(3);
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
const scanSlot = await res.json();
|
|
1011
|
+
reservedScanId = scanSlot.scanId;
|
|
1012
|
+
info(
|
|
1013
|
+
`Scan reserved (${scanSlot.remaining}/${scanSlot.limit} remaining this month)`
|
|
1014
|
+
);
|
|
1015
|
+
} catch (err) {
|
|
1016
|
+
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
1017
|
+
console.error(`Failed to connect to backend: ${msg}`);
|
|
1018
|
+
setExitCode(3);
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
const evaluator = new BackendEvaluator(
|
|
1022
|
+
credentials.apiUrl,
|
|
1023
|
+
credentials.apiKey
|
|
1024
|
+
);
|
|
1025
|
+
const cliOverrides = {};
|
|
1026
|
+
if (opts.endpoint) {
|
|
1027
|
+
cliOverrides.endpoint = {
|
|
1028
|
+
url: opts.endpoint,
|
|
1029
|
+
auth: buildAuthFromFlags(opts),
|
|
1030
|
+
request: {
|
|
1031
|
+
method: "POST",
|
|
1032
|
+
body: opts.requestBody ? JSON.parse(opts.requestBody) : { message: "{{prompt}}" }
|
|
1033
|
+
},
|
|
1034
|
+
response: {
|
|
1035
|
+
field: opts.responseField ?? "response"
|
|
1036
|
+
}
|
|
1037
|
+
};
|
|
1038
|
+
} else if (opts.provider || opts.model) {
|
|
1039
|
+
cliOverrides.target = {
|
|
1040
|
+
provider: opts.provider,
|
|
1041
|
+
model: opts.model
|
|
1042
|
+
};
|
|
1043
|
+
}
|
|
1044
|
+
if (opts.preset) cliOverrides.preset = opts.preset;
|
|
1045
|
+
if (opts.systemPrompt) cliOverrides.systemPrompt = opts.systemPrompt;
|
|
1046
|
+
if (opts.crescendo) cliOverrides.includeCrescendo = true;
|
|
1047
|
+
if (opts.minScore !== void 0 || opts.failOnCritical) {
|
|
1048
|
+
cliOverrides.thresholds = {
|
|
1049
|
+
...opts.minScore !== void 0 ? { minScore: opts.minScore } : {},
|
|
1050
|
+
...opts.failOnCritical ? { failOnCritical: true } : {}
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
if (opts.output || opts.outputFile) {
|
|
1054
|
+
cliOverrides.output = {
|
|
1055
|
+
...opts.output ? { format: opts.output } : {},
|
|
1056
|
+
...opts.outputFile ? { file: opts.outputFile } : {}
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
const config = loadConfig({
|
|
1060
|
+
configPath: opts.config,
|
|
1061
|
+
cliOverrides
|
|
1062
|
+
});
|
|
1063
|
+
if (opts.loadCustomTests) {
|
|
1064
|
+
try {
|
|
1065
|
+
const remoteTests = await fetchCustomTests(
|
|
1066
|
+
credentials.apiUrl,
|
|
1067
|
+
credentials.apiKey
|
|
1068
|
+
);
|
|
1069
|
+
if (remoteTests.length === 0) {
|
|
1070
|
+
info(
|
|
1071
|
+
`No custom tests found in your dashboard. Create them at ${DEFAULT_API_URL}/tests`
|
|
1072
|
+
);
|
|
1073
|
+
} else {
|
|
1074
|
+
info(`Loaded ${remoteTests.length} custom test(s) from dashboard`);
|
|
1075
|
+
config.customTests = [...config.customTests ?? [], ...remoteTests];
|
|
1076
|
+
}
|
|
1077
|
+
} catch (err) {
|
|
1078
|
+
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
1079
|
+
console.error(`Failed to fetch custom tests: ${msg}`);
|
|
1080
|
+
setExitCode(3);
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
if (opts.systemPromptFile) {
|
|
1085
|
+
config.systemPrompt = opts.systemPromptFile;
|
|
1086
|
+
}
|
|
1087
|
+
const systemPrompt = loadSystemPrompt(config);
|
|
1088
|
+
const presetForScan = config.preset ?? "standard";
|
|
1089
|
+
const includeCrescendo = config.includeCrescendo ?? config.preset === "full";
|
|
1090
|
+
const fetched = await fetchAttacks({
|
|
1091
|
+
apiUrl: credentials.apiUrl,
|
|
1092
|
+
apiKey: credentials.apiKey,
|
|
1093
|
+
preset: presetForScan,
|
|
1094
|
+
includeMultiTurn: includeCrescendo
|
|
1095
|
+
});
|
|
1096
|
+
const multiTurnNote = fetched.multiTurnAttacks.length > 0 ? ` + ${fetched.multiTurnAttacks.length} multi-turn` : "";
|
|
1097
|
+
info(
|
|
1098
|
+
`Loaded ${fetched.attacks.length} attacks${multiTurnNote} from ${fetched.source} (catalog ${fetched.catalogVersion})`
|
|
1099
|
+
);
|
|
1100
|
+
const scanOptions = {
|
|
1101
|
+
target: buildTargetConfig(config),
|
|
1102
|
+
systemPrompt,
|
|
1103
|
+
preset: presetForScan,
|
|
1104
|
+
attacks: fetched.attacks,
|
|
1105
|
+
multiTurnAttacks: fetched.multiTurnAttacks,
|
|
1106
|
+
includeCrescendo,
|
|
1107
|
+
customTests: config.customTests,
|
|
1108
|
+
concurrency: config.concurrency,
|
|
1109
|
+
delayMs: config.delayMs,
|
|
1110
|
+
attackTimeoutMs: config.attackTimeoutMs,
|
|
1111
|
+
maxConsecutiveFailures: config.maxConsecutiveFailures,
|
|
1112
|
+
evaluator
|
|
1113
|
+
};
|
|
1114
|
+
const onEvent = opts.progress !== false ? handleEvent : void 0;
|
|
1115
|
+
const scanner = new Scanner(scanOptions, onEvent);
|
|
1116
|
+
const report = await scanner.run();
|
|
1117
|
+
try {
|
|
1118
|
+
if (!reservedScanId) {
|
|
1119
|
+
throw new Error(
|
|
1120
|
+
"Missing scan reservation id (server returned no scanId)."
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
await uploadReportToBackend(report, credentials, reservedScanId);
|
|
1124
|
+
} catch (err) {
|
|
1125
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1126
|
+
console.warn(`Warning: Could not save scan to dashboard: ${msg}`);
|
|
1127
|
+
}
|
|
1128
|
+
let thresholdResult = evaluateThresholds(report, config.thresholds);
|
|
1129
|
+
if (!opts.ci) {
|
|
1130
|
+
printSummary(report, thresholdResult);
|
|
1131
|
+
}
|
|
1132
|
+
let complianceResult;
|
|
1133
|
+
if (opts.euAiAct || opts.complianceThreshold !== void 0) {
|
|
1134
|
+
const testOutcomes = report.attacks.filter((a) => !a.error).map((a) => ({
|
|
1135
|
+
testId: a.id,
|
|
1136
|
+
testName: a.name,
|
|
1137
|
+
vulnerable: a.vulnerable,
|
|
1138
|
+
severity: a.severity
|
|
1139
|
+
}));
|
|
1140
|
+
complianceResult = calculateComplianceScore(testOutcomes);
|
|
1141
|
+
if (!opts.ci) {
|
|
1142
|
+
console.log(formatComplianceTerminal(complianceResult, report.target));
|
|
1143
|
+
}
|
|
1144
|
+
if (opts.complianceThreshold !== void 0 && complianceResult.overallScore < opts.complianceThreshold) {
|
|
1145
|
+
thresholdResult = {
|
|
1146
|
+
passed: false,
|
|
1147
|
+
reasons: [
|
|
1148
|
+
...thresholdResult.reasons,
|
|
1149
|
+
`Compliance score ${complianceResult.overallScore}% is below minimum threshold ${opts.complianceThreshold}%`
|
|
1150
|
+
]
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
const format = config.output?.format ?? opts.output;
|
|
1155
|
+
const outputFile = config.output?.file ?? opts.outputFile;
|
|
1156
|
+
if (format || outputFile) {
|
|
1157
|
+
if (format === "pdf") {
|
|
1158
|
+
const pdfPath = outputFile ?? `shield-report-${report.id.slice(0, 8)}.pdf`;
|
|
1159
|
+
const pdfBuffer = await generatePdf(report);
|
|
1160
|
+
writeFileSync4(resolve3(pdfPath), pdfBuffer);
|
|
1161
|
+
info(`PDF report written to ${pdfPath}`);
|
|
1162
|
+
} else {
|
|
1163
|
+
let content;
|
|
1164
|
+
switch (format) {
|
|
1165
|
+
case "sarif":
|
|
1166
|
+
content = formatSarif(report);
|
|
1167
|
+
break;
|
|
1168
|
+
case "markdown":
|
|
1169
|
+
content = formatMarkdown(report, thresholdResult, complianceResult);
|
|
1170
|
+
break;
|
|
1171
|
+
case "json":
|
|
1172
|
+
default:
|
|
1173
|
+
content = formatJson({
|
|
1174
|
+
report,
|
|
1175
|
+
thresholds: thresholdResult,
|
|
1176
|
+
compliance: complianceResult
|
|
1177
|
+
});
|
|
1178
|
+
break;
|
|
1179
|
+
}
|
|
1180
|
+
if (outputFile) {
|
|
1181
|
+
writeFileSync4(resolve3(outputFile), content, "utf-8");
|
|
1182
|
+
info(`Report written to ${outputFile}`);
|
|
1183
|
+
} else {
|
|
1184
|
+
console.log(content);
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
if (opts.writeJson) {
|
|
1189
|
+
const jsonPath = resolve3(opts.writeJson);
|
|
1190
|
+
const reportForDisk = opts.includeFullPayloads ? report : sanitizeReportForDisk(report);
|
|
1191
|
+
const jsonContent = formatJson({
|
|
1192
|
+
report: reportForDisk,
|
|
1193
|
+
thresholds: thresholdResult,
|
|
1194
|
+
compliance: complianceResult
|
|
1195
|
+
});
|
|
1196
|
+
writeFileSync4(jsonPath, jsonContent, "utf-8");
|
|
1197
|
+
info(
|
|
1198
|
+
`JSON copy written to ${jsonPath}${opts.includeFullPayloads ? " (full payloads)" : " (sanitized)"}`
|
|
1199
|
+
);
|
|
1200
|
+
}
|
|
1201
|
+
setExitCode(thresholdResult.passed ? 0 : 1);
|
|
1202
|
+
return;
|
|
1203
|
+
} catch (error) {
|
|
1204
|
+
if (error instanceof ConfigError) {
|
|
1205
|
+
console.error(`Configuration error: ${error.message}`);
|
|
1206
|
+
setExitCode(2);
|
|
1207
|
+
return;
|
|
1208
|
+
}
|
|
1209
|
+
const msg = error instanceof Error ? error.message : "Unknown error";
|
|
1210
|
+
console.error(`Runtime error: ${msg}`);
|
|
1211
|
+
setExitCode(3);
|
|
1212
|
+
return;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// src/commands/tests.ts
|
|
1217
|
+
async function testsCommand(opts) {
|
|
1218
|
+
let localTests = [];
|
|
1219
|
+
let remoteTests = [];
|
|
1220
|
+
try {
|
|
1221
|
+
const config = loadConfig({ configPath: opts.config });
|
|
1222
|
+
if (config.customTests?.length) {
|
|
1223
|
+
localTests = config.customTests;
|
|
1224
|
+
}
|
|
1225
|
+
} catch {
|
|
1226
|
+
}
|
|
1227
|
+
if (opts.remote) {
|
|
1228
|
+
const stored = loadCredentials();
|
|
1229
|
+
if (!stored) {
|
|
1230
|
+
console.error("Not logged in. Run: shield-llm login --key <your-key>\n");
|
|
1231
|
+
setExitCode(2);
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
const apiUrl = resolveApiUrl(stored.apiUrl, opts.apiUrl);
|
|
1235
|
+
try {
|
|
1236
|
+
remoteTests = await fetchCustomTests(apiUrl, stored.apiKey);
|
|
1237
|
+
} catch (err) {
|
|
1238
|
+
const msg = err instanceof Error ? err.message : "Unknown error";
|
|
1239
|
+
console.error(`Failed to fetch remote tests: ${msg}`);
|
|
1240
|
+
setExitCode(3);
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
const hasLocal = localTests.length > 0;
|
|
1245
|
+
const hasRemote = remoteTests.length > 0;
|
|
1246
|
+
if (!hasLocal && !hasRemote) {
|
|
1247
|
+
console.log("\nNo custom tests found.\n");
|
|
1248
|
+
if (!opts.remote) {
|
|
1249
|
+
console.log(" Tip: use --remote to check your Shield LLM dashboard\n");
|
|
1250
|
+
}
|
|
1251
|
+
console.log(" Create tests via:");
|
|
1252
|
+
console.log(
|
|
1253
|
+
" shield-llm init Add tests to shield.config.json"
|
|
1254
|
+
);
|
|
1255
|
+
console.log(
|
|
1256
|
+
` ${DEFAULT_API_URL}/tests Create tests on the dashboard
|
|
1257
|
+
`
|
|
1258
|
+
);
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
if (hasLocal) {
|
|
1262
|
+
console.log(`
|
|
1263
|
+
Local tests (shield.config.json): ${localTests.length}
|
|
1264
|
+
`);
|
|
1265
|
+
for (const test of localTests) {
|
|
1266
|
+
console.log(` ${test.name}`);
|
|
1267
|
+
console.log(` Prompt: ${truncate(test.prompt, 80)}`);
|
|
1268
|
+
console.log(` Criteria: ${truncate(test.acceptanceCriteria, 80)}
|
|
1269
|
+
`);
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
if (hasRemote) {
|
|
1273
|
+
console.log(`
|
|
1274
|
+
Dashboard tests (remote): ${remoteTests.length}
|
|
1275
|
+
`);
|
|
1276
|
+
for (const test of remoteTests) {
|
|
1277
|
+
console.log(` ${test.name}`);
|
|
1278
|
+
console.log(` Prompt: ${truncate(test.prompt, 80)}`);
|
|
1279
|
+
console.log(` Criteria: ${truncate(test.acceptanceCriteria, 80)}
|
|
1280
|
+
`);
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
const total = localTests.length + remoteTests.length;
|
|
1284
|
+
console.log(
|
|
1285
|
+
`Total: ${total} custom test(s) will be added to your next scan.
|
|
1286
|
+
`
|
|
1287
|
+
);
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// src/commands/validate.ts
|
|
1291
|
+
function validateCommand(opts) {
|
|
1292
|
+
try {
|
|
1293
|
+
const config = loadConfig({ configPath: opts.config });
|
|
1294
|
+
console.log("Configuration is valid.");
|
|
1295
|
+
if (config.endpoint) {
|
|
1296
|
+
console.log(` Mode: HTTP Endpoint`);
|
|
1297
|
+
console.log(` URL: ${config.endpoint.url}`);
|
|
1298
|
+
console.log(` Auth: ${config.endpoint.auth?.type ?? "none"}`);
|
|
1299
|
+
console.log(` Response field: ${config.endpoint.response.field}`);
|
|
1300
|
+
} else {
|
|
1301
|
+
console.log(` Mode: LLM Provider`);
|
|
1302
|
+
console.log(
|
|
1303
|
+
` Target: ${config.target.provider}/${config.target.model ?? "default"}`
|
|
1304
|
+
);
|
|
1305
|
+
}
|
|
1306
|
+
console.log(
|
|
1307
|
+
` Judge: ${config.judge?.provider ?? "mistral"}/${config.judge?.model ?? "mistral-small-latest"}`
|
|
1308
|
+
);
|
|
1309
|
+
console.log(` Preset: ${config.preset ?? "standard"}`);
|
|
1310
|
+
if (config.customTests?.length) {
|
|
1311
|
+
console.log(` Custom tests: ${config.customTests.length}`);
|
|
1312
|
+
}
|
|
1313
|
+
setExitCode(0);
|
|
1314
|
+
return;
|
|
1315
|
+
} catch (error) {
|
|
1316
|
+
if (error instanceof ConfigError) {
|
|
1317
|
+
console.error(`Invalid configuration: ${error.message}`);
|
|
1318
|
+
setExitCode(2);
|
|
1319
|
+
return;
|
|
1320
|
+
}
|
|
1321
|
+
throw error;
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
// src/cli.ts
|
|
1326
|
+
function reportFatal(label, err) {
|
|
1327
|
+
console.error(`
|
|
1328
|
+
${label}:`);
|
|
1329
|
+
if (err instanceof Error) {
|
|
1330
|
+
console.error(err.stack ?? err.message);
|
|
1331
|
+
} else {
|
|
1332
|
+
console.error(String(err));
|
|
1333
|
+
}
|
|
1334
|
+
console.error(
|
|
1335
|
+
"\nThis is an unexpected error. Please rerun with the stack trace above attached if you report it."
|
|
1336
|
+
);
|
|
1337
|
+
process.exit(3);
|
|
1338
|
+
}
|
|
1339
|
+
process.on("unhandledRejection", (reason) => {
|
|
1340
|
+
reportFatal("Unhandled promise rejection", reason);
|
|
1341
|
+
});
|
|
1342
|
+
process.on("uncaughtException", (err) => {
|
|
1343
|
+
reportFatal("Uncaught exception", err);
|
|
1344
|
+
});
|
|
1345
|
+
var program = new Command();
|
|
1346
|
+
program.exitOverride();
|
|
1347
|
+
program.name("shield-llm").description("AI chatbot security scanner \u2014 automated red teaming for LLMs").version("0.1.0");
|
|
1348
|
+
program.command("scan").description("Run a security scan against a target chatbot").option("-c, --config <path>", "Path to shield.config.json").option("-p, --provider <name>", "Target provider (openai, mistral)").option("-m, --model <name>", "Target model name").option("-e, --endpoint <url>", "Target chatbot HTTP endpoint URL").option("--auth <type>", "Auth type: bearer, api-key, oauth2").option("--token <value>", "Bearer token or API key value").option("--auth-header <name>", "Custom auth header name (for api-key auth)").option(
|
|
1349
|
+
"--response-field <path>",
|
|
1350
|
+
"Dot-notation path to response text in JSON (default: response)"
|
|
1351
|
+
).option(
|
|
1352
|
+
"--request-body <json>",
|
|
1353
|
+
"JSON request body template (use {{prompt}} placeholder)"
|
|
1354
|
+
).option("--preset <name>", "Scan preset (dev, quick, owasp, standard, full)").option("--system-prompt <text>", "System prompt for target chatbot").option("--system-prompt-file <path>", "Path to system prompt file").option("--crescendo", "Include crescendo (multi-turn) attacks").option("--min-score <n>", "Minimum score threshold", parseFloat).option("--fail-on-critical", "Fail if any CRITICAL vulnerability found").option("-o, --output <format>", "Output format (json, markdown, sarif, pdf)").option("--output-file <path>", "Write report to file").option(
|
|
1355
|
+
"--write-json <path>",
|
|
1356
|
+
"Always write a parseable JSON copy here, regardless of --output (useful for CI to extract score/grade alongside SARIF)"
|
|
1357
|
+
).option(
|
|
1358
|
+
"--include-full-payloads",
|
|
1359
|
+
"Keep full chatbot responses / LLM reasoning in the on-disk JSON (default: truncate to protect against artifact leaks in public CI runs)"
|
|
1360
|
+
).option("--no-progress", "Disable progress spinner").option(
|
|
1361
|
+
"--ci",
|
|
1362
|
+
"CI mode: JSON to stdout, no spinner. Exit 0=pass, 1=threshold, 2=config/auth, 3=infra"
|
|
1363
|
+
).option(
|
|
1364
|
+
"--load-custom-tests",
|
|
1365
|
+
"Fetch custom tests from your Shield LLM dashboard"
|
|
1366
|
+
).option("--eu-ai-act", "Show EU AI Act compliance assessment after scan").option(
|
|
1367
|
+
"--compliance-threshold <n>",
|
|
1368
|
+
"Minimum compliance score (0-100)",
|
|
1369
|
+
parseFloat
|
|
1370
|
+
).option(
|
|
1371
|
+
"--api-url <url>",
|
|
1372
|
+
"Backend API URL (overrides SHIELD_API_URL env var and stored credentials)"
|
|
1373
|
+
).action(scanCommand);
|
|
1374
|
+
program.command("init").description("Generate a shield.config.json interactively").action(initCommand);
|
|
1375
|
+
program.command("attacks").description("List all available attacks (fetched from your backend)").option("--severity <level>", "Filter by severity").option("--api-url <url>", "Backend API URL override").action(attacksCommand);
|
|
1376
|
+
program.command("presets").description("List available scan presets (fetched from your backend)").option("--api-url <url>", "Backend API URL override").action(presetsCommand);
|
|
1377
|
+
program.command("tests").description("List custom tests (local config and/or remote dashboard)").option("-c, --config <path>", "Path to shield.config.json").option("--remote", "Fetch custom tests from your Shield LLM dashboard").option(
|
|
1378
|
+
"--api-url <url>",
|
|
1379
|
+
"Backend API URL (overrides SHIELD_API_URL env var)"
|
|
1380
|
+
).action(testsCommand);
|
|
1381
|
+
program.command("validate").description("Validate a shield.config.json file").option("-c, --config <path>", "Path to config file").action(validateCommand);
|
|
1382
|
+
program.command("report <scanId>").description("Fetch a past scan report from your dashboard").option("-o, --output <format>", "Output format (json, markdown)", "json").option("--output-file <path>", "Write report to file").option("--api-url <url>", "Backend API URL override").action(reportCommand);
|
|
1383
|
+
program.command("login").description("Authenticate with your Shield LLM API key").requiredOption("--key <api-key>", "Your API key (sk_shield_...)").option(
|
|
1384
|
+
"--api-url <url>",
|
|
1385
|
+
"Backend API URL (defaults to SHIELD_API_URL env var or https://shield-llm.com)"
|
|
1386
|
+
).action(loginCommand);
|
|
1387
|
+
program.command("logout").description("Remove stored API key credentials").action(logoutCommand);
|
|
1388
|
+
try {
|
|
1389
|
+
program.parse();
|
|
1390
|
+
} catch (err) {
|
|
1391
|
+
if (err instanceof CommanderError) {
|
|
1392
|
+
const code = err.code === "commander.helpDisplayed" || err.code === "commander.version" ? 0 : 2;
|
|
1393
|
+
process.exit(code);
|
|
1394
|
+
}
|
|
1395
|
+
reportFatal("Error", err);
|
|
1396
|
+
}
|