trickle-cli 0.1.190 → 0.1.192
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/compliance.d.ts +17 -0
- package/dist/commands/compliance.js +251 -0
- package/dist/commands/security.d.ts +1 -6
- package/dist/commands/security.js +95 -1
- package/dist/index.js +7 -0
- package/package.json +1 -1
- package/src/commands/compliance.ts +262 -0
- package/src/commands/security.ts +101 -3
- package/src/index.ts +7 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
3
|
+
*
|
|
4
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
5
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
6
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
7
|
+
* - Timestamped event log
|
|
8
|
+
* - Risk classification
|
|
9
|
+
* - Security scan results
|
|
10
|
+
* - Data processing summary
|
|
11
|
+
*
|
|
12
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
13
|
+
*/
|
|
14
|
+
export declare function generateComplianceReport(opts: {
|
|
15
|
+
json?: boolean;
|
|
16
|
+
out?: string;
|
|
17
|
+
}): void;
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
4
|
+
*
|
|
5
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
6
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
7
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
8
|
+
* - Timestamped event log
|
|
9
|
+
* - Risk classification
|
|
10
|
+
* - Security scan results
|
|
11
|
+
* - Data processing summary
|
|
12
|
+
*
|
|
13
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
14
|
+
*/
|
|
15
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
18
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
19
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
20
|
+
}
|
|
21
|
+
Object.defineProperty(o, k2, desc);
|
|
22
|
+
}) : (function(o, m, k, k2) {
|
|
23
|
+
if (k2 === undefined) k2 = k;
|
|
24
|
+
o[k2] = m[k];
|
|
25
|
+
}));
|
|
26
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
27
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
28
|
+
}) : function(o, v) {
|
|
29
|
+
o["default"] = v;
|
|
30
|
+
});
|
|
31
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
32
|
+
var ownKeys = function(o) {
|
|
33
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
34
|
+
var ar = [];
|
|
35
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
36
|
+
return ar;
|
|
37
|
+
};
|
|
38
|
+
return ownKeys(o);
|
|
39
|
+
};
|
|
40
|
+
return function (mod) {
|
|
41
|
+
if (mod && mod.__esModule) return mod;
|
|
42
|
+
var result = {};
|
|
43
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
44
|
+
__setModuleDefault(result, mod);
|
|
45
|
+
return result;
|
|
46
|
+
};
|
|
47
|
+
})();
|
|
48
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
49
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
50
|
+
};
|
|
51
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
|
+
exports.generateComplianceReport = generateComplianceReport;
|
|
53
|
+
const fs = __importStar(require("fs"));
|
|
54
|
+
const path = __importStar(require("path"));
|
|
55
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
56
|
+
function readJsonl(fp) {
|
|
57
|
+
if (!fs.existsSync(fp))
|
|
58
|
+
return [];
|
|
59
|
+
return fs.readFileSync(fp, 'utf-8').split('\n').filter(Boolean)
|
|
60
|
+
.map(l => { try {
|
|
61
|
+
return JSON.parse(l);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return null;
|
|
65
|
+
} }).filter(Boolean);
|
|
66
|
+
}
|
|
67
|
+
function generateComplianceReport(opts) {
|
|
68
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
69
|
+
if (!fs.existsSync(dir)) {
|
|
70
|
+
console.log(chalk_1.default.yellow(' No .trickle/ data. Run trickle first.'));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const llmCalls = readJsonl(path.join(dir, 'llm.jsonl'));
|
|
74
|
+
const agentEvents = readJsonl(path.join(dir, 'agents.jsonl'));
|
|
75
|
+
const mcpCalls = readJsonl(path.join(dir, 'mcp.jsonl'));
|
|
76
|
+
const errors = readJsonl(path.join(dir, 'errors.jsonl'));
|
|
77
|
+
const observations = readJsonl(path.join(dir, 'observations.jsonl'));
|
|
78
|
+
const calltrace = readJsonl(path.join(dir, 'calltrace.jsonl'));
|
|
79
|
+
// Build decision lineage — chronological event log
|
|
80
|
+
const lineage = [];
|
|
81
|
+
for (const e of agentEvents) {
|
|
82
|
+
lineage.push({
|
|
83
|
+
timestamp: e.timestamp || 0,
|
|
84
|
+
event: `agent:${e.event}`,
|
|
85
|
+
description: buildAgentDescription(e),
|
|
86
|
+
data: { framework: e.framework, tool: e.tool, chain: e.chain },
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
for (const c of llmCalls) {
|
|
90
|
+
lineage.push({
|
|
91
|
+
timestamp: c.timestamp || 0,
|
|
92
|
+
event: 'llm:call',
|
|
93
|
+
description: `${c.provider}/${c.model}: ${(c.inputPreview || '').substring(0, 80)} → ${(c.outputPreview || '').substring(0, 80)}`,
|
|
94
|
+
data: { model: c.model, tokens: c.totalTokens, cost: c.estimatedCostUsd, error: c.error },
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
for (const m of mcpCalls) {
|
|
98
|
+
if (m.tool === '__list_tools')
|
|
99
|
+
continue;
|
|
100
|
+
lineage.push({
|
|
101
|
+
timestamp: m.timestamp || 0,
|
|
102
|
+
event: `mcp:${m.direction}`,
|
|
103
|
+
description: `${m.tool}(${JSON.stringify(m.args || {}).substring(0, 60)}) → ${(m.resultPreview || '').substring(0, 60)}`,
|
|
104
|
+
data: { tool: m.tool, direction: m.direction, isError: m.isError },
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
lineage.sort((a, b) => a.timestamp - b.timestamp);
|
|
108
|
+
// Data processing summary
|
|
109
|
+
const providers = [...new Set(llmCalls.map(c => c.provider).filter(Boolean))];
|
|
110
|
+
const models = [...new Set(llmCalls.map(c => `${c.provider}/${c.model}`).filter(Boolean))];
|
|
111
|
+
const totalTokens = llmCalls.reduce((s, c) => s + (c.totalTokens || 0), 0);
|
|
112
|
+
const totalCost = llmCalls.reduce((s, c) => s + (c.estimatedCostUsd || 0), 0);
|
|
113
|
+
const agentTools = [...new Set(agentEvents.filter(e => e.tool).map(e => e.tool))];
|
|
114
|
+
const mcpTools = [...new Set(mcpCalls.filter(c => c.tool && c.tool !== '__list_tools').map(c => c.tool))];
|
|
115
|
+
const dataSources = [...new Set(observations.map(o => o.module).filter(Boolean))];
|
|
116
|
+
// Security scan
|
|
117
|
+
let securityFindings = [];
|
|
118
|
+
try {
|
|
119
|
+
const { runSecurityScan } = require('./security');
|
|
120
|
+
const origLog = console.log;
|
|
121
|
+
console.log = () => { };
|
|
122
|
+
const result = runSecurityScan({ dir });
|
|
123
|
+
console.log = origLog;
|
|
124
|
+
securityFindings = result.findings.map((f) => ({
|
|
125
|
+
severity: f.severity, category: f.category,
|
|
126
|
+
message: f.message, evidence: (f.evidence || '').substring(0, 100),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
catch { }
|
|
130
|
+
// Eval score
|
|
131
|
+
let evalScore = null;
|
|
132
|
+
try {
|
|
133
|
+
const { evalCommand } = require('./eval');
|
|
134
|
+
// We can't easily call evalCommand and get the result, so build a lightweight score
|
|
135
|
+
const completionRate = agentEvents.filter(e => e.event === 'crew_end').length /
|
|
136
|
+
Math.max(1, agentEvents.filter(e => e.event === 'crew_start').length);
|
|
137
|
+
const errorRate = errors.length / Math.max(1, lineage.length);
|
|
138
|
+
evalScore = {
|
|
139
|
+
overall: Math.round(Math.max(0, (1 - errorRate) * completionRate * 100)),
|
|
140
|
+
grade: completionRate >= 0.9 && errorRate < 0.1 ? 'A' : completionRate >= 0.7 ? 'B' : 'C',
|
|
141
|
+
dimensions: { completion: Math.round(completionRate * 100), errorRate: Math.round((1 - errorRate) * 100) },
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
catch { }
|
|
145
|
+
// Risk classification
|
|
146
|
+
const riskFactors = [];
|
|
147
|
+
if (llmCalls.length > 0)
|
|
148
|
+
riskFactors.push('Uses AI/LLM for decision-making');
|
|
149
|
+
if (agentEvents.length > 0)
|
|
150
|
+
riskFactors.push('Autonomous agent workflow');
|
|
151
|
+
if (agentTools.length > 0)
|
|
152
|
+
riskFactors.push(`Executes ${agentTools.length} tools autonomously`);
|
|
153
|
+
if (securityFindings.filter(f => f.severity === 'critical').length > 0)
|
|
154
|
+
riskFactors.push('Critical security findings');
|
|
155
|
+
if (errors.length > 0)
|
|
156
|
+
riskFactors.push(`${errors.length} runtime errors`);
|
|
157
|
+
const riskLevel = riskFactors.length >= 3 ? 'high' : riskFactors.length >= 1 ? 'medium' : 'low';
|
|
158
|
+
// Human oversight
|
|
159
|
+
const permissionEvents = agentEvents.filter(e => e.event === 'permission_request' || (e.tool || '').toLowerCase().includes('approval'));
|
|
160
|
+
const escalationEvents = agentEvents.filter(e => e.event?.includes('error') || e.event === 'crew_error');
|
|
161
|
+
const report = {
|
|
162
|
+
meta: {
|
|
163
|
+
generatedAt: new Date().toISOString(),
|
|
164
|
+
trickleVersion: 'CLI 0.1.191',
|
|
165
|
+
framework: [...new Set(agentEvents.map(e => e.framework).filter(Boolean))].join(', ') || 'N/A',
|
|
166
|
+
dataDir: dir,
|
|
167
|
+
},
|
|
168
|
+
riskClassification: { level: riskLevel, factors: riskFactors },
|
|
169
|
+
decisionLineage: lineage,
|
|
170
|
+
dataProcessing: {
|
|
171
|
+
llmProviders: providers, modelsUsed: models,
|
|
172
|
+
totalLlmCalls: llmCalls.length, totalTokens, estimatedCost: Math.round(totalCost * 10000) / 10000,
|
|
173
|
+
toolsUsed: agentTools, mcpToolsUsed: mcpTools,
|
|
174
|
+
dataSourcesAccessed: dataSources.slice(0, 20),
|
|
175
|
+
},
|
|
176
|
+
securityFindings,
|
|
177
|
+
evalScore,
|
|
178
|
+
humanOversight: {
|
|
179
|
+
hasHumanInLoop: permissionEvents.length > 0,
|
|
180
|
+
approvalCheckpoints: permissionEvents.length,
|
|
181
|
+
escalationEvents: escalationEvents.length,
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
// Output
|
|
185
|
+
if (opts.out) {
|
|
186
|
+
fs.writeFileSync(opts.out, JSON.stringify(report, null, 2), 'utf-8');
|
|
187
|
+
console.log(chalk_1.default.green(` Compliance report written to ${opts.out}`));
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (opts.json) {
|
|
191
|
+
console.log(JSON.stringify(report, null, 2));
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
// Pretty print
|
|
195
|
+
console.log('');
|
|
196
|
+
console.log(chalk_1.default.bold(' trickle audit --compliance'));
|
|
197
|
+
console.log(chalk_1.default.gray(' ' + '─'.repeat(60)));
|
|
198
|
+
const riskColor = riskLevel === 'high' ? chalk_1.default.red : riskLevel === 'medium' ? chalk_1.default.yellow : chalk_1.default.green;
|
|
199
|
+
console.log(` Risk: ${riskColor(riskLevel.toUpperCase())} (${riskFactors.length} factors)`);
|
|
200
|
+
for (const f of riskFactors)
|
|
201
|
+
console.log(chalk_1.default.gray(` • ${f}`));
|
|
202
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
203
|
+
console.log(chalk_1.default.bold(' Data Processing'));
|
|
204
|
+
console.log(` LLM providers: ${providers.join(', ') || 'none'}`);
|
|
205
|
+
console.log(` Models: ${models.join(', ') || 'none'}`);
|
|
206
|
+
console.log(` Calls: ${llmCalls.length} Tokens: ${totalTokens} Cost: $${totalCost.toFixed(4)}`);
|
|
207
|
+
console.log(` Tools: ${agentTools.join(', ') || 'none'}`);
|
|
208
|
+
if (mcpTools.length > 0)
|
|
209
|
+
console.log(` MCP tools: ${mcpTools.join(', ')}`);
|
|
210
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
211
|
+
console.log(chalk_1.default.bold(' Decision Lineage'));
|
|
212
|
+
console.log(` ${lineage.length} events recorded`);
|
|
213
|
+
for (const e of lineage.slice(0, 10)) {
|
|
214
|
+
const ts = new Date(e.timestamp).toISOString().substring(11, 23);
|
|
215
|
+
console.log(chalk_1.default.gray(` ${ts}`) + ` ${e.event.padEnd(20)} ${e.description.substring(0, 60)}`);
|
|
216
|
+
}
|
|
217
|
+
if (lineage.length > 10)
|
|
218
|
+
console.log(chalk_1.default.gray(` ... and ${lineage.length - 10} more`));
|
|
219
|
+
if (securityFindings.length > 0) {
|
|
220
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
221
|
+
console.log(chalk_1.default.bold(' Security'));
|
|
222
|
+
const crit = securityFindings.filter(f => f.severity === 'critical').length;
|
|
223
|
+
const warn = securityFindings.filter(f => f.severity === 'warning').length;
|
|
224
|
+
console.log(` ${chalk_1.default.red(String(crit))} critical, ${chalk_1.default.yellow(String(warn))} warnings`);
|
|
225
|
+
}
|
|
226
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
227
|
+
console.log(chalk_1.default.bold(' Human Oversight'));
|
|
228
|
+
console.log(` Human-in-the-loop: ${report.humanOversight.hasHumanInLoop ? chalk_1.default.green('Yes') : chalk_1.default.yellow('No')}`);
|
|
229
|
+
console.log(` Approval checkpoints: ${report.humanOversight.approvalCheckpoints}`);
|
|
230
|
+
console.log(` Escalation events: ${report.humanOversight.escalationEvents}`);
|
|
231
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
232
|
+
console.log(chalk_1.default.gray(' Export: trickle audit --compliance --json > audit-report.json'));
|
|
233
|
+
console.log(chalk_1.default.gray(' Export: trickle audit --compliance -o audit-report.json'));
|
|
234
|
+
console.log('');
|
|
235
|
+
}
|
|
236
|
+
function buildAgentDescription(e) {
|
|
237
|
+
const parts = [];
|
|
238
|
+
if (e.chain)
|
|
239
|
+
parts.push(e.chain);
|
|
240
|
+
if (e.tool)
|
|
241
|
+
parts.push(`tool:${e.tool}`);
|
|
242
|
+
if (e.toolInput)
|
|
243
|
+
parts.push(`input:${String(e.toolInput).substring(0, 40)}`);
|
|
244
|
+
if (e.output)
|
|
245
|
+
parts.push(`output:${String(e.output).substring(0, 40)}`);
|
|
246
|
+
if (e.error)
|
|
247
|
+
parts.push(`error:${String(e.error).substring(0, 40)}`);
|
|
248
|
+
if (e.thought)
|
|
249
|
+
parts.push(`thought:${String(e.thought).substring(0, 40)}`);
|
|
250
|
+
return parts.join(' | ') || e.event || '?';
|
|
251
|
+
}
|
|
@@ -21,12 +21,7 @@ interface SecurityFinding {
|
|
|
21
21
|
}
|
|
22
22
|
export interface SecurityResult {
|
|
23
23
|
findings: SecurityFinding[];
|
|
24
|
-
scanned:
|
|
25
|
-
variables: number;
|
|
26
|
-
queries: number;
|
|
27
|
-
logs: number;
|
|
28
|
-
observations: number;
|
|
29
|
-
};
|
|
24
|
+
scanned: Record<string, number>;
|
|
30
25
|
summary: {
|
|
31
26
|
critical: number;
|
|
32
27
|
warning: number;
|
|
@@ -154,6 +154,90 @@ function runSecurityScan(opts) {
|
|
|
154
154
|
if (o.sampleOutput)
|
|
155
155
|
findings.push(...scanValue(o.sampleOutput, 'function_output', `${o.module}.${o.functionName}`));
|
|
156
156
|
}
|
|
157
|
+
// ── Agent Security: The "Lethal Trifecta" ──
|
|
158
|
+
// Scan LLM calls for prompt injection and data exfiltration
|
|
159
|
+
const llmCalls = readJsonl(path.join(trickleDir, 'llm.jsonl'));
|
|
160
|
+
for (const c of llmCalls) {
|
|
161
|
+
// Prompt injection patterns in LLM inputs
|
|
162
|
+
const input = String(c.inputPreview || '').toLowerCase();
|
|
163
|
+
const INJECTION_PATTERNS = [
|
|
164
|
+
{ pattern: /ignore\s+(all\s+)?previous\s+instructions/i, name: 'Instruction override' },
|
|
165
|
+
{ pattern: /you\s+are\s+now\s+a\s+/i, name: 'Role hijacking' },
|
|
166
|
+
{ pattern: /system\s*:\s*you\s+are/i, name: 'System prompt injection' },
|
|
167
|
+
{ pattern: /\bdo\s+not\s+follow\s+(any|the)\s+(previous|above)/i, name: 'Instruction bypass' },
|
|
168
|
+
{ pattern: /forget\s+(all|everything|your)\s+(previous|prior|instructions)/i, name: 'Memory wipe attempt' },
|
|
169
|
+
{ pattern: /pretend\s+you\s+(are|have)\s+(no|unrestricted)/i, name: 'Jailbreak attempt' },
|
|
170
|
+
];
|
|
171
|
+
for (const inj of INJECTION_PATTERNS) {
|
|
172
|
+
if (inj.pattern.test(c.inputPreview || '') || inj.pattern.test(c.systemPrompt || '')) {
|
|
173
|
+
findings.push({
|
|
174
|
+
severity: 'critical', category: 'prompt_injection',
|
|
175
|
+
message: `${inj.name} detected in LLM input`,
|
|
176
|
+
source: 'llm_call', location: c.model || 'unknown',
|
|
177
|
+
evidence: (c.inputPreview || '').substring(0, 100),
|
|
178
|
+
});
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
// Secrets in LLM outputs (data exfiltration)
|
|
183
|
+
const output = String(c.outputPreview || '');
|
|
184
|
+
if (output) {
|
|
185
|
+
const outputFindings = scanValue(output, 'llm_output', `${c.provider}/${c.model}`);
|
|
186
|
+
for (const f of outputFindings) {
|
|
187
|
+
f.category = 'data_exfiltration';
|
|
188
|
+
f.message = `LLM output contains ${f.message.toLowerCase()}`;
|
|
189
|
+
findings.push(f);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
// Secrets in LLM inputs
|
|
193
|
+
const inputStr = String(c.inputPreview || '');
|
|
194
|
+
if (inputStr) {
|
|
195
|
+
const inputFindings = scanValue(inputStr, 'llm_input', `${c.provider}/${c.model}`);
|
|
196
|
+
for (const f of inputFindings) {
|
|
197
|
+
f.message = `Secret passed to LLM: ${f.message}`;
|
|
198
|
+
findings.push(f);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// Scan agent events for unauthorized tool calls
|
|
203
|
+
const agentEvents = readJsonl(path.join(trickleDir, 'agents.jsonl'));
|
|
204
|
+
const toolErrors = agentEvents.filter(e => e.event === 'tool_error');
|
|
205
|
+
const toolStarts = agentEvents.filter(e => e.event === 'tool_start');
|
|
206
|
+
// Detect privilege escalation: agent calling dangerous tools
|
|
207
|
+
const DANGEROUS_TOOLS = ['Bash', 'bash', 'shell', 'exec', 'eval', 'rm', 'sudo', 'chmod', 'kill'];
|
|
208
|
+
for (const t of toolStarts) {
|
|
209
|
+
const toolName = String(t.tool || '');
|
|
210
|
+
if (DANGEROUS_TOOLS.some(d => toolName.toLowerCase().includes(d.toLowerCase()))) {
|
|
211
|
+
// Check if tool input contains dangerous commands
|
|
212
|
+
const toolInput = String(t.toolInput || '').toLowerCase();
|
|
213
|
+
if (toolInput.includes('rm -rf') || toolInput.includes('sudo') || toolInput.includes('chmod 777') ||
|
|
214
|
+
toolInput.includes('curl') && toolInput.includes('|') || toolInput.includes('wget') && toolInput.includes('|')) {
|
|
215
|
+
findings.push({
|
|
216
|
+
severity: 'critical', category: 'privilege_escalation',
|
|
217
|
+
message: `Agent executed dangerous command via ${toolName}`,
|
|
218
|
+
source: 'agent_tool', location: t.framework || 'agent',
|
|
219
|
+
evidence: (t.toolInput || '').substring(0, 100),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// Scan MCP tool calls for secrets in args/responses
|
|
225
|
+
const mcpCalls = readJsonl(path.join(trickleDir, 'mcp.jsonl'));
|
|
226
|
+
for (const m of mcpCalls) {
|
|
227
|
+
if (m.args) {
|
|
228
|
+
const argsStr = typeof m.args === 'string' ? m.args : JSON.stringify(m.args);
|
|
229
|
+
const argsFindings = scanValue(argsStr, 'mcp_tool_args', `MCP: ${m.tool}`);
|
|
230
|
+
findings.push(...argsFindings);
|
|
231
|
+
}
|
|
232
|
+
if (m.resultPreview) {
|
|
233
|
+
const resultFindings = scanValue(m.resultPreview, 'mcp_tool_result', `MCP: ${m.tool}`);
|
|
234
|
+
for (const f of resultFindings) {
|
|
235
|
+
f.category = 'data_exfiltration';
|
|
236
|
+
f.message = `MCP tool response contains ${f.message.toLowerCase()}`;
|
|
237
|
+
findings.push(f);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
157
241
|
// Deduplicate
|
|
158
242
|
const seen = new Set();
|
|
159
243
|
const deduped = findings.filter(f => {
|
|
@@ -168,6 +252,9 @@ function runSecurityScan(opts) {
|
|
|
168
252
|
warning: deduped.filter(f => f.severity === 'warning').length,
|
|
169
253
|
info: deduped.filter(f => f.severity === 'info').length,
|
|
170
254
|
};
|
|
255
|
+
scanned.llmCalls = llmCalls.length;
|
|
256
|
+
scanned.agentEvents = agentEvents.length;
|
|
257
|
+
scanned.mcpCalls = mcpCalls.length;
|
|
171
258
|
const result = { findings: deduped, scanned, summary };
|
|
172
259
|
if (opts?.json) {
|
|
173
260
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -177,7 +264,14 @@ function runSecurityScan(opts) {
|
|
|
177
264
|
console.log('');
|
|
178
265
|
console.log(chalk_1.default.bold(' trickle security'));
|
|
179
266
|
console.log(chalk_1.default.gray(' ' + '─'.repeat(50)));
|
|
180
|
-
|
|
267
|
+
const scanParts = [`${scanned.variables} vars`, `${scanned.queries} queries`, `${scanned.logs} logs`, `${scanned.observations} functions`];
|
|
268
|
+
if (scanned.llmCalls)
|
|
269
|
+
scanParts.push(`${scanned.llmCalls} LLM calls`);
|
|
270
|
+
if (scanned.agentEvents)
|
|
271
|
+
scanParts.push(`${scanned.agentEvents} agent events`);
|
|
272
|
+
if (scanned.mcpCalls)
|
|
273
|
+
scanParts.push(`${scanned.mcpCalls} MCP calls`);
|
|
274
|
+
console.log(chalk_1.default.gray(` Scanned: ${scanParts.join(', ')}`));
|
|
181
275
|
if (deduped.length === 0) {
|
|
182
276
|
console.log(chalk_1.default.green(' No security issues found. ✓'));
|
|
183
277
|
}
|
package/dist/index.js
CHANGED
|
@@ -396,8 +396,15 @@ program
|
|
|
396
396
|
.option("--json", "Output raw JSON (for CI integration)")
|
|
397
397
|
.option("--fail-on-error", "Exit 1 if any errors are found")
|
|
398
398
|
.option("--fail-on-warning", "Exit 1 if any errors or warnings are found")
|
|
399
|
+
.option("--compliance", "Generate compliance audit report (EU AI Act / Colorado AI Act)")
|
|
400
|
+
.option("-o, --out <file>", "Write compliance report to file")
|
|
399
401
|
.option("--local", "Read from local .trickle/observations.jsonl instead of backend")
|
|
400
402
|
.action(async (opts) => {
|
|
403
|
+
if (opts.compliance) {
|
|
404
|
+
const { generateComplianceReport } = await Promise.resolve().then(() => __importStar(require("./commands/compliance")));
|
|
405
|
+
generateComplianceReport({ json: opts.json, out: opts.out });
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
401
408
|
await (0, audit_1.auditCommand)(opts);
|
|
402
409
|
});
|
|
403
410
|
// trickle capture <method> <url>
|
package/package.json
CHANGED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
3
|
+
*
|
|
4
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
5
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
6
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
7
|
+
* - Timestamped event log
|
|
8
|
+
* - Risk classification
|
|
9
|
+
* - Security scan results
|
|
10
|
+
* - Data processing summary
|
|
11
|
+
*
|
|
12
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
import chalk from 'chalk';
|
|
18
|
+
|
|
19
|
+
function readJsonl(fp: string): any[] {
|
|
20
|
+
if (!fs.existsSync(fp)) return [];
|
|
21
|
+
return fs.readFileSync(fp, 'utf-8').split('\n').filter(Boolean)
|
|
22
|
+
.map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
interface ComplianceReport {
|
|
26
|
+
meta: {
|
|
27
|
+
generatedAt: string;
|
|
28
|
+
trickleVersion: string;
|
|
29
|
+
framework: string;
|
|
30
|
+
dataDir: string;
|
|
31
|
+
};
|
|
32
|
+
riskClassification: {
|
|
33
|
+
level: 'high' | 'medium' | 'low';
|
|
34
|
+
factors: string[];
|
|
35
|
+
};
|
|
36
|
+
decisionLineage: Array<{
|
|
37
|
+
timestamp: number;
|
|
38
|
+
event: string;
|
|
39
|
+
description: string;
|
|
40
|
+
data?: Record<string, unknown>;
|
|
41
|
+
}>;
|
|
42
|
+
dataProcessing: {
|
|
43
|
+
llmProviders: string[];
|
|
44
|
+
modelsUsed: string[];
|
|
45
|
+
totalLlmCalls: number;
|
|
46
|
+
totalTokens: number;
|
|
47
|
+
estimatedCost: number;
|
|
48
|
+
toolsUsed: string[];
|
|
49
|
+
mcpToolsUsed: string[];
|
|
50
|
+
dataSourcesAccessed: string[];
|
|
51
|
+
};
|
|
52
|
+
securityFindings: Array<{
|
|
53
|
+
severity: string;
|
|
54
|
+
category: string;
|
|
55
|
+
message: string;
|
|
56
|
+
evidence: string;
|
|
57
|
+
}>;
|
|
58
|
+
evalScore: {
|
|
59
|
+
overall: number;
|
|
60
|
+
grade: string;
|
|
61
|
+
dimensions: Record<string, number>;
|
|
62
|
+
} | null;
|
|
63
|
+
humanOversight: {
|
|
64
|
+
hasHumanInLoop: boolean;
|
|
65
|
+
approvalCheckpoints: number;
|
|
66
|
+
escalationEvents: number;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function generateComplianceReport(opts: { json?: boolean; out?: string }): void {
|
|
71
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
72
|
+
|
|
73
|
+
if (!fs.existsSync(dir)) {
|
|
74
|
+
console.log(chalk.yellow(' No .trickle/ data. Run trickle first.'));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const llmCalls = readJsonl(path.join(dir, 'llm.jsonl'));
|
|
79
|
+
const agentEvents = readJsonl(path.join(dir, 'agents.jsonl'));
|
|
80
|
+
const mcpCalls = readJsonl(path.join(dir, 'mcp.jsonl'));
|
|
81
|
+
const errors = readJsonl(path.join(dir, 'errors.jsonl'));
|
|
82
|
+
const observations = readJsonl(path.join(dir, 'observations.jsonl'));
|
|
83
|
+
const calltrace = readJsonl(path.join(dir, 'calltrace.jsonl'));
|
|
84
|
+
|
|
85
|
+
// Build decision lineage — chronological event log
|
|
86
|
+
const lineage: ComplianceReport['decisionLineage'] = [];
|
|
87
|
+
|
|
88
|
+
for (const e of agentEvents) {
|
|
89
|
+
lineage.push({
|
|
90
|
+
timestamp: e.timestamp || 0,
|
|
91
|
+
event: `agent:${e.event}`,
|
|
92
|
+
description: buildAgentDescription(e),
|
|
93
|
+
data: { framework: e.framework, tool: e.tool, chain: e.chain },
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
for (const c of llmCalls) {
|
|
98
|
+
lineage.push({
|
|
99
|
+
timestamp: c.timestamp || 0,
|
|
100
|
+
event: 'llm:call',
|
|
101
|
+
description: `${c.provider}/${c.model}: ${(c.inputPreview || '').substring(0, 80)} → ${(c.outputPreview || '').substring(0, 80)}`,
|
|
102
|
+
data: { model: c.model, tokens: c.totalTokens, cost: c.estimatedCostUsd, error: c.error },
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const m of mcpCalls) {
|
|
107
|
+
if (m.tool === '__list_tools') continue;
|
|
108
|
+
lineage.push({
|
|
109
|
+
timestamp: m.timestamp || 0,
|
|
110
|
+
event: `mcp:${m.direction}`,
|
|
111
|
+
description: `${m.tool}(${JSON.stringify(m.args || {}).substring(0, 60)}) → ${(m.resultPreview || '').substring(0, 60)}`,
|
|
112
|
+
data: { tool: m.tool, direction: m.direction, isError: m.isError },
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
lineage.sort((a, b) => a.timestamp - b.timestamp);
|
|
117
|
+
|
|
118
|
+
// Data processing summary
|
|
119
|
+
const providers = [...new Set(llmCalls.map(c => c.provider).filter(Boolean))];
|
|
120
|
+
const models = [...new Set(llmCalls.map(c => `${c.provider}/${c.model}`).filter(Boolean))];
|
|
121
|
+
const totalTokens = llmCalls.reduce((s: number, c: any) => s + (c.totalTokens || 0), 0);
|
|
122
|
+
const totalCost = llmCalls.reduce((s: number, c: any) => s + (c.estimatedCostUsd || 0), 0);
|
|
123
|
+
const agentTools = [...new Set(agentEvents.filter(e => e.tool).map(e => e.tool))];
|
|
124
|
+
const mcpTools = [...new Set(mcpCalls.filter(c => c.tool && c.tool !== '__list_tools').map(c => c.tool))];
|
|
125
|
+
const dataSources = [...new Set(observations.map(o => o.module).filter(Boolean))];
|
|
126
|
+
|
|
127
|
+
// Security scan
|
|
128
|
+
let securityFindings: ComplianceReport['securityFindings'] = [];
|
|
129
|
+
try {
|
|
130
|
+
const { runSecurityScan } = require('./security');
|
|
131
|
+
const origLog = console.log;
|
|
132
|
+
console.log = () => {};
|
|
133
|
+
const result = runSecurityScan({ dir });
|
|
134
|
+
console.log = origLog;
|
|
135
|
+
securityFindings = result.findings.map((f: any) => ({
|
|
136
|
+
severity: f.severity, category: f.category,
|
|
137
|
+
message: f.message, evidence: (f.evidence || '').substring(0, 100),
|
|
138
|
+
}));
|
|
139
|
+
} catch {}
|
|
140
|
+
|
|
141
|
+
// Eval score
|
|
142
|
+
let evalScore: ComplianceReport['evalScore'] = null;
|
|
143
|
+
try {
|
|
144
|
+
const { evalCommand } = require('./eval');
|
|
145
|
+
// We can't easily call evalCommand and get the result, so build a lightweight score
|
|
146
|
+
const completionRate = agentEvents.filter(e => e.event === 'crew_end').length /
|
|
147
|
+
Math.max(1, agentEvents.filter(e => e.event === 'crew_start').length);
|
|
148
|
+
const errorRate = errors.length / Math.max(1, lineage.length);
|
|
149
|
+
evalScore = {
|
|
150
|
+
overall: Math.round(Math.max(0, (1 - errorRate) * completionRate * 100)),
|
|
151
|
+
grade: completionRate >= 0.9 && errorRate < 0.1 ? 'A' : completionRate >= 0.7 ? 'B' : 'C',
|
|
152
|
+
dimensions: { completion: Math.round(completionRate * 100), errorRate: Math.round((1 - errorRate) * 100) },
|
|
153
|
+
};
|
|
154
|
+
} catch {}
|
|
155
|
+
|
|
156
|
+
// Risk classification
|
|
157
|
+
const riskFactors: string[] = [];
|
|
158
|
+
if (llmCalls.length > 0) riskFactors.push('Uses AI/LLM for decision-making');
|
|
159
|
+
if (agentEvents.length > 0) riskFactors.push('Autonomous agent workflow');
|
|
160
|
+
if (agentTools.length > 0) riskFactors.push(`Executes ${agentTools.length} tools autonomously`);
|
|
161
|
+
if (securityFindings.filter(f => f.severity === 'critical').length > 0) riskFactors.push('Critical security findings');
|
|
162
|
+
if (errors.length > 0) riskFactors.push(`${errors.length} runtime errors`);
|
|
163
|
+
const riskLevel: 'high' | 'medium' | 'low' = riskFactors.length >= 3 ? 'high' : riskFactors.length >= 1 ? 'medium' : 'low';
|
|
164
|
+
|
|
165
|
+
// Human oversight
|
|
166
|
+
const permissionEvents = agentEvents.filter(e =>
|
|
167
|
+
e.event === 'permission_request' || (e.tool || '').toLowerCase().includes('approval'));
|
|
168
|
+
const escalationEvents = agentEvents.filter(e =>
|
|
169
|
+
e.event?.includes('error') || e.event === 'crew_error');
|
|
170
|
+
|
|
171
|
+
const report: ComplianceReport = {
|
|
172
|
+
meta: {
|
|
173
|
+
generatedAt: new Date().toISOString(),
|
|
174
|
+
trickleVersion: 'CLI 0.1.191',
|
|
175
|
+
framework: [...new Set(agentEvents.map(e => e.framework).filter(Boolean))].join(', ') || 'N/A',
|
|
176
|
+
dataDir: dir,
|
|
177
|
+
},
|
|
178
|
+
riskClassification: { level: riskLevel, factors: riskFactors },
|
|
179
|
+
decisionLineage: lineage,
|
|
180
|
+
dataProcessing: {
|
|
181
|
+
llmProviders: providers, modelsUsed: models,
|
|
182
|
+
totalLlmCalls: llmCalls.length, totalTokens, estimatedCost: Math.round(totalCost * 10000) / 10000,
|
|
183
|
+
toolsUsed: agentTools, mcpToolsUsed: mcpTools,
|
|
184
|
+
dataSourcesAccessed: dataSources.slice(0, 20),
|
|
185
|
+
},
|
|
186
|
+
securityFindings,
|
|
187
|
+
evalScore,
|
|
188
|
+
humanOversight: {
|
|
189
|
+
hasHumanInLoop: permissionEvents.length > 0,
|
|
190
|
+
approvalCheckpoints: permissionEvents.length,
|
|
191
|
+
escalationEvents: escalationEvents.length,
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
// Output
|
|
196
|
+
if (opts.out) {
|
|
197
|
+
fs.writeFileSync(opts.out, JSON.stringify(report, null, 2), 'utf-8');
|
|
198
|
+
console.log(chalk.green(` Compliance report written to ${opts.out}`));
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (opts.json) {
|
|
203
|
+
console.log(JSON.stringify(report, null, 2));
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Pretty print
|
|
208
|
+
console.log('');
|
|
209
|
+
console.log(chalk.bold(' trickle audit --compliance'));
|
|
210
|
+
console.log(chalk.gray(' ' + '─'.repeat(60)));
|
|
211
|
+
|
|
212
|
+
const riskColor = riskLevel === 'high' ? chalk.red : riskLevel === 'medium' ? chalk.yellow : chalk.green;
|
|
213
|
+
console.log(` Risk: ${riskColor(riskLevel.toUpperCase())} (${riskFactors.length} factors)`);
|
|
214
|
+
for (const f of riskFactors) console.log(chalk.gray(` • ${f}`));
|
|
215
|
+
|
|
216
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
217
|
+
console.log(chalk.bold(' Data Processing'));
|
|
218
|
+
console.log(` LLM providers: ${providers.join(', ') || 'none'}`);
|
|
219
|
+
console.log(` Models: ${models.join(', ') || 'none'}`);
|
|
220
|
+
console.log(` Calls: ${llmCalls.length} Tokens: ${totalTokens} Cost: $${totalCost.toFixed(4)}`);
|
|
221
|
+
console.log(` Tools: ${agentTools.join(', ') || 'none'}`);
|
|
222
|
+
if (mcpTools.length > 0) console.log(` MCP tools: ${mcpTools.join(', ')}`);
|
|
223
|
+
|
|
224
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
225
|
+
console.log(chalk.bold(' Decision Lineage'));
|
|
226
|
+
console.log(` ${lineage.length} events recorded`);
|
|
227
|
+
for (const e of lineage.slice(0, 10)) {
|
|
228
|
+
const ts = new Date(e.timestamp).toISOString().substring(11, 23);
|
|
229
|
+
console.log(chalk.gray(` ${ts}`) + ` ${e.event.padEnd(20)} ${e.description.substring(0, 60)}`);
|
|
230
|
+
}
|
|
231
|
+
if (lineage.length > 10) console.log(chalk.gray(` ... and ${lineage.length - 10} more`));
|
|
232
|
+
|
|
233
|
+
if (securityFindings.length > 0) {
|
|
234
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
235
|
+
console.log(chalk.bold(' Security'));
|
|
236
|
+
const crit = securityFindings.filter(f => f.severity === 'critical').length;
|
|
237
|
+
const warn = securityFindings.filter(f => f.severity === 'warning').length;
|
|
238
|
+
console.log(` ${chalk.red(String(crit))} critical, ${chalk.yellow(String(warn))} warnings`);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
242
|
+
console.log(chalk.bold(' Human Oversight'));
|
|
243
|
+
console.log(` Human-in-the-loop: ${report.humanOversight.hasHumanInLoop ? chalk.green('Yes') : chalk.yellow('No')}`);
|
|
244
|
+
console.log(` Approval checkpoints: ${report.humanOversight.approvalCheckpoints}`);
|
|
245
|
+
console.log(` Escalation events: ${report.humanOversight.escalationEvents}`);
|
|
246
|
+
|
|
247
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
248
|
+
console.log(chalk.gray(' Export: trickle audit --compliance --json > audit-report.json'));
|
|
249
|
+
console.log(chalk.gray(' Export: trickle audit --compliance -o audit-report.json'));
|
|
250
|
+
console.log('');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function buildAgentDescription(e: any): string {
|
|
254
|
+
const parts: string[] = [];
|
|
255
|
+
if (e.chain) parts.push(e.chain);
|
|
256
|
+
if (e.tool) parts.push(`tool:${e.tool}`);
|
|
257
|
+
if (e.toolInput) parts.push(`input:${String(e.toolInput).substring(0, 40)}`);
|
|
258
|
+
if (e.output) parts.push(`output:${String(e.output).substring(0, 40)}`);
|
|
259
|
+
if (e.error) parts.push(`error:${String(e.error).substring(0, 40)}`);
|
|
260
|
+
if (e.thought) parts.push(`thought:${String(e.thought).substring(0, 40)}`);
|
|
261
|
+
return parts.join(' | ') || e.event || '?';
|
|
262
|
+
}
|
package/src/commands/security.ts
CHANGED
|
@@ -82,14 +82,14 @@ function scanValue(value: unknown, source: string, location: string): SecurityFi
|
|
|
82
82
|
|
|
83
83
|
export interface SecurityResult {
|
|
84
84
|
findings: SecurityFinding[];
|
|
85
|
-
scanned:
|
|
85
|
+
scanned: Record<string, number>;
|
|
86
86
|
summary: { critical: number; warning: number; info: number };
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
export function runSecurityScan(opts?: { dir?: string; json?: boolean }): SecurityResult {
|
|
90
90
|
const trickleDir = opts?.dir || process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
91
91
|
const findings: SecurityFinding[] = [];
|
|
92
|
-
const scanned = { variables: 0, queries: 0, logs: 0, observations: 0 };
|
|
92
|
+
const scanned: Record<string, number> = { variables: 0, queries: 0, logs: 0, observations: 0 };
|
|
93
93
|
|
|
94
94
|
// Scan variables
|
|
95
95
|
const variables = readJsonl(path.join(trickleDir, 'variables.jsonl'));
|
|
@@ -133,6 +133,97 @@ export function runSecurityScan(opts?: { dir?: string; json?: boolean }): Securi
|
|
|
133
133
|
if (o.sampleOutput) findings.push(...scanValue(o.sampleOutput, 'function_output', `${o.module}.${o.functionName}`));
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
// ── Agent Security: The "Lethal Trifecta" ──
|
|
137
|
+
|
|
138
|
+
// Scan LLM calls for prompt injection and data exfiltration
|
|
139
|
+
const llmCalls = readJsonl(path.join(trickleDir, 'llm.jsonl'));
|
|
140
|
+
for (const c of llmCalls) {
|
|
141
|
+
// Prompt injection patterns in LLM inputs
|
|
142
|
+
const input = String(c.inputPreview || '').toLowerCase();
|
|
143
|
+
const INJECTION_PATTERNS = [
|
|
144
|
+
{ pattern: /ignore\s+(all\s+)?previous\s+instructions/i, name: 'Instruction override' },
|
|
145
|
+
{ pattern: /you\s+are\s+now\s+a\s+/i, name: 'Role hijacking' },
|
|
146
|
+
{ pattern: /system\s*:\s*you\s+are/i, name: 'System prompt injection' },
|
|
147
|
+
{ pattern: /\bdo\s+not\s+follow\s+(any|the)\s+(previous|above)/i, name: 'Instruction bypass' },
|
|
148
|
+
{ pattern: /forget\s+(all|everything|your)\s+(previous|prior|instructions)/i, name: 'Memory wipe attempt' },
|
|
149
|
+
{ pattern: /pretend\s+you\s+(are|have)\s+(no|unrestricted)/i, name: 'Jailbreak attempt' },
|
|
150
|
+
];
|
|
151
|
+
for (const inj of INJECTION_PATTERNS) {
|
|
152
|
+
if (inj.pattern.test(c.inputPreview || '') || inj.pattern.test(c.systemPrompt || '')) {
|
|
153
|
+
findings.push({
|
|
154
|
+
severity: 'critical', category: 'prompt_injection',
|
|
155
|
+
message: `${inj.name} detected in LLM input`,
|
|
156
|
+
source: 'llm_call', location: c.model || 'unknown',
|
|
157
|
+
evidence: (c.inputPreview || '').substring(0, 100),
|
|
158
|
+
});
|
|
159
|
+
break;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Secrets in LLM outputs (data exfiltration)
|
|
164
|
+
const output = String(c.outputPreview || '');
|
|
165
|
+
if (output) {
|
|
166
|
+
const outputFindings = scanValue(output, 'llm_output', `${c.provider}/${c.model}`);
|
|
167
|
+
for (const f of outputFindings) {
|
|
168
|
+
f.category = 'data_exfiltration';
|
|
169
|
+
f.message = `LLM output contains ${f.message.toLowerCase()}`;
|
|
170
|
+
findings.push(f);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// Secrets in LLM inputs
|
|
175
|
+
const inputStr = String(c.inputPreview || '');
|
|
176
|
+
if (inputStr) {
|
|
177
|
+
const inputFindings = scanValue(inputStr, 'llm_input', `${c.provider}/${c.model}`);
|
|
178
|
+
for (const f of inputFindings) {
|
|
179
|
+
f.message = `Secret passed to LLM: ${f.message}`;
|
|
180
|
+
findings.push(f);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Scan agent events for unauthorized tool calls
|
|
186
|
+
const agentEvents = readJsonl(path.join(trickleDir, 'agents.jsonl'));
|
|
187
|
+
const toolErrors = agentEvents.filter(e => e.event === 'tool_error');
|
|
188
|
+
const toolStarts = agentEvents.filter(e => e.event === 'tool_start');
|
|
189
|
+
|
|
190
|
+
// Detect privilege escalation: agent calling dangerous tools
|
|
191
|
+
const DANGEROUS_TOOLS = ['Bash', 'bash', 'shell', 'exec', 'eval', 'rm', 'sudo', 'chmod', 'kill'];
|
|
192
|
+
for (const t of toolStarts) {
|
|
193
|
+
const toolName = String(t.tool || '');
|
|
194
|
+
if (DANGEROUS_TOOLS.some(d => toolName.toLowerCase().includes(d.toLowerCase()))) {
|
|
195
|
+
// Check if tool input contains dangerous commands
|
|
196
|
+
const toolInput = String(t.toolInput || '').toLowerCase();
|
|
197
|
+
if (toolInput.includes('rm -rf') || toolInput.includes('sudo') || toolInput.includes('chmod 777') ||
|
|
198
|
+
toolInput.includes('curl') && toolInput.includes('|') || toolInput.includes('wget') && toolInput.includes('|')) {
|
|
199
|
+
findings.push({
|
|
200
|
+
severity: 'critical', category: 'privilege_escalation',
|
|
201
|
+
message: `Agent executed dangerous command via ${toolName}`,
|
|
202
|
+
source: 'agent_tool', location: t.framework || 'agent',
|
|
203
|
+
evidence: (t.toolInput || '').substring(0, 100),
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Scan MCP tool calls for secrets in args/responses
|
|
210
|
+
const mcpCalls = readJsonl(path.join(trickleDir, 'mcp.jsonl'));
|
|
211
|
+
for (const m of mcpCalls) {
|
|
212
|
+
if (m.args) {
|
|
213
|
+
const argsStr = typeof m.args === 'string' ? m.args : JSON.stringify(m.args);
|
|
214
|
+
const argsFindings = scanValue(argsStr, 'mcp_tool_args', `MCP: ${m.tool}`);
|
|
215
|
+
findings.push(...argsFindings);
|
|
216
|
+
}
|
|
217
|
+
if (m.resultPreview) {
|
|
218
|
+
const resultFindings = scanValue(m.resultPreview, 'mcp_tool_result', `MCP: ${m.tool}`);
|
|
219
|
+
for (const f of resultFindings) {
|
|
220
|
+
f.category = 'data_exfiltration';
|
|
221
|
+
f.message = `MCP tool response contains ${f.message.toLowerCase()}`;
|
|
222
|
+
findings.push(f);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
136
227
|
// Deduplicate
|
|
137
228
|
const seen = new Set<string>();
|
|
138
229
|
const deduped = findings.filter(f => {
|
|
@@ -148,6 +239,9 @@ export function runSecurityScan(opts?: { dir?: string; json?: boolean }): Securi
|
|
|
148
239
|
info: deduped.filter(f => f.severity === 'info').length,
|
|
149
240
|
};
|
|
150
241
|
|
|
242
|
+
scanned.llmCalls = llmCalls.length;
|
|
243
|
+
scanned.agentEvents = agentEvents.length;
|
|
244
|
+
scanned.mcpCalls = mcpCalls.length;
|
|
151
245
|
const result: SecurityResult = { findings: deduped, scanned, summary };
|
|
152
246
|
|
|
153
247
|
if (opts?.json) {
|
|
@@ -159,7 +253,11 @@ export function runSecurityScan(opts?: { dir?: string; json?: boolean }): Securi
|
|
|
159
253
|
console.log('');
|
|
160
254
|
console.log(chalk.bold(' trickle security'));
|
|
161
255
|
console.log(chalk.gray(' ' + '─'.repeat(50)));
|
|
162
|
-
|
|
256
|
+
const scanParts = [`${scanned.variables} vars`, `${scanned.queries} queries`, `${scanned.logs} logs`, `${scanned.observations} functions`];
|
|
257
|
+
if (scanned.llmCalls) scanParts.push(`${scanned.llmCalls} LLM calls`);
|
|
258
|
+
if (scanned.agentEvents) scanParts.push(`${scanned.agentEvents} agent events`);
|
|
259
|
+
if (scanned.mcpCalls) scanParts.push(`${scanned.mcpCalls} MCP calls`);
|
|
260
|
+
console.log(chalk.gray(` Scanned: ${scanParts.join(', ')}`));
|
|
163
261
|
|
|
164
262
|
if (deduped.length === 0) {
|
|
165
263
|
console.log(chalk.green(' No security issues found. ✓'));
|
package/src/index.ts
CHANGED
|
@@ -384,8 +384,15 @@ program
|
|
|
384
384
|
.option("--json", "Output raw JSON (for CI integration)")
|
|
385
385
|
.option("--fail-on-error", "Exit 1 if any errors are found")
|
|
386
386
|
.option("--fail-on-warning", "Exit 1 if any errors or warnings are found")
|
|
387
|
+
.option("--compliance", "Generate compliance audit report (EU AI Act / Colorado AI Act)")
|
|
388
|
+
.option("-o, --out <file>", "Write compliance report to file")
|
|
387
389
|
.option("--local", "Read from local .trickle/observations.jsonl instead of backend")
|
|
388
390
|
.action(async (opts) => {
|
|
391
|
+
if (opts.compliance) {
|
|
392
|
+
const { generateComplianceReport } = await import("./commands/compliance");
|
|
393
|
+
generateComplianceReport({ json: opts.json, out: opts.out });
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
389
396
|
await auditCommand(opts);
|
|
390
397
|
});
|
|
391
398
|
|