trickle-cli 0.1.191 → 0.1.192
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/compliance.d.ts +17 -0
- package/dist/commands/compliance.js +251 -0
- package/dist/index.js +7 -0
- package/package.json +1 -1
- package/src/commands/compliance.ts +262 -0
- package/src/index.ts +7 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
3
|
+
*
|
|
4
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
5
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
6
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
7
|
+
* - Timestamped event log
|
|
8
|
+
* - Risk classification
|
|
9
|
+
* - Security scan results
|
|
10
|
+
* - Data processing summary
|
|
11
|
+
*
|
|
12
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
13
|
+
*/
|
|
14
|
+
export declare function generateComplianceReport(opts: {
|
|
15
|
+
json?: boolean;
|
|
16
|
+
out?: string;
|
|
17
|
+
}): void;
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
4
|
+
*
|
|
5
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
6
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
7
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
8
|
+
* - Timestamped event log
|
|
9
|
+
* - Risk classification
|
|
10
|
+
* - Security scan results
|
|
11
|
+
* - Data processing summary
|
|
12
|
+
*
|
|
13
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
14
|
+
*/
|
|
15
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
18
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
19
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
20
|
+
}
|
|
21
|
+
Object.defineProperty(o, k2, desc);
|
|
22
|
+
}) : (function(o, m, k, k2) {
|
|
23
|
+
if (k2 === undefined) k2 = k;
|
|
24
|
+
o[k2] = m[k];
|
|
25
|
+
}));
|
|
26
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
27
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
28
|
+
}) : function(o, v) {
|
|
29
|
+
o["default"] = v;
|
|
30
|
+
});
|
|
31
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
32
|
+
var ownKeys = function(o) {
|
|
33
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
34
|
+
var ar = [];
|
|
35
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
36
|
+
return ar;
|
|
37
|
+
};
|
|
38
|
+
return ownKeys(o);
|
|
39
|
+
};
|
|
40
|
+
return function (mod) {
|
|
41
|
+
if (mod && mod.__esModule) return mod;
|
|
42
|
+
var result = {};
|
|
43
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
44
|
+
__setModuleDefault(result, mod);
|
|
45
|
+
return result;
|
|
46
|
+
};
|
|
47
|
+
})();
|
|
48
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
49
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
50
|
+
};
|
|
51
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
52
|
+
exports.generateComplianceReport = generateComplianceReport;
|
|
53
|
+
const fs = __importStar(require("fs"));
|
|
54
|
+
const path = __importStar(require("path"));
|
|
55
|
+
const chalk_1 = __importDefault(require("chalk"));
|
|
56
|
+
function readJsonl(fp) {
|
|
57
|
+
if (!fs.existsSync(fp))
|
|
58
|
+
return [];
|
|
59
|
+
return fs.readFileSync(fp, 'utf-8').split('\n').filter(Boolean)
|
|
60
|
+
.map(l => { try {
|
|
61
|
+
return JSON.parse(l);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return null;
|
|
65
|
+
} }).filter(Boolean);
|
|
66
|
+
}
|
|
67
|
+
function generateComplianceReport(opts) {
|
|
68
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
69
|
+
if (!fs.existsSync(dir)) {
|
|
70
|
+
console.log(chalk_1.default.yellow(' No .trickle/ data. Run trickle first.'));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const llmCalls = readJsonl(path.join(dir, 'llm.jsonl'));
|
|
74
|
+
const agentEvents = readJsonl(path.join(dir, 'agents.jsonl'));
|
|
75
|
+
const mcpCalls = readJsonl(path.join(dir, 'mcp.jsonl'));
|
|
76
|
+
const errors = readJsonl(path.join(dir, 'errors.jsonl'));
|
|
77
|
+
const observations = readJsonl(path.join(dir, 'observations.jsonl'));
|
|
78
|
+
const calltrace = readJsonl(path.join(dir, 'calltrace.jsonl'));
|
|
79
|
+
// Build decision lineage — chronological event log
|
|
80
|
+
const lineage = [];
|
|
81
|
+
for (const e of agentEvents) {
|
|
82
|
+
lineage.push({
|
|
83
|
+
timestamp: e.timestamp || 0,
|
|
84
|
+
event: `agent:${e.event}`,
|
|
85
|
+
description: buildAgentDescription(e),
|
|
86
|
+
data: { framework: e.framework, tool: e.tool, chain: e.chain },
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
for (const c of llmCalls) {
|
|
90
|
+
lineage.push({
|
|
91
|
+
timestamp: c.timestamp || 0,
|
|
92
|
+
event: 'llm:call',
|
|
93
|
+
description: `${c.provider}/${c.model}: ${(c.inputPreview || '').substring(0, 80)} → ${(c.outputPreview || '').substring(0, 80)}`,
|
|
94
|
+
data: { model: c.model, tokens: c.totalTokens, cost: c.estimatedCostUsd, error: c.error },
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
for (const m of mcpCalls) {
|
|
98
|
+
if (m.tool === '__list_tools')
|
|
99
|
+
continue;
|
|
100
|
+
lineage.push({
|
|
101
|
+
timestamp: m.timestamp || 0,
|
|
102
|
+
event: `mcp:${m.direction}`,
|
|
103
|
+
description: `${m.tool}(${JSON.stringify(m.args || {}).substring(0, 60)}) → ${(m.resultPreview || '').substring(0, 60)}`,
|
|
104
|
+
data: { tool: m.tool, direction: m.direction, isError: m.isError },
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
lineage.sort((a, b) => a.timestamp - b.timestamp);
|
|
108
|
+
// Data processing summary
|
|
109
|
+
const providers = [...new Set(llmCalls.map(c => c.provider).filter(Boolean))];
|
|
110
|
+
const models = [...new Set(llmCalls.map(c => `${c.provider}/${c.model}`).filter(Boolean))];
|
|
111
|
+
const totalTokens = llmCalls.reduce((s, c) => s + (c.totalTokens || 0), 0);
|
|
112
|
+
const totalCost = llmCalls.reduce((s, c) => s + (c.estimatedCostUsd || 0), 0);
|
|
113
|
+
const agentTools = [...new Set(agentEvents.filter(e => e.tool).map(e => e.tool))];
|
|
114
|
+
const mcpTools = [...new Set(mcpCalls.filter(c => c.tool && c.tool !== '__list_tools').map(c => c.tool))];
|
|
115
|
+
const dataSources = [...new Set(observations.map(o => o.module).filter(Boolean))];
|
|
116
|
+
// Security scan
|
|
117
|
+
let securityFindings = [];
|
|
118
|
+
try {
|
|
119
|
+
const { runSecurityScan } = require('./security');
|
|
120
|
+
const origLog = console.log;
|
|
121
|
+
console.log = () => { };
|
|
122
|
+
const result = runSecurityScan({ dir });
|
|
123
|
+
console.log = origLog;
|
|
124
|
+
securityFindings = result.findings.map((f) => ({
|
|
125
|
+
severity: f.severity, category: f.category,
|
|
126
|
+
message: f.message, evidence: (f.evidence || '').substring(0, 100),
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
catch { }
|
|
130
|
+
// Eval score
|
|
131
|
+
let evalScore = null;
|
|
132
|
+
try {
|
|
133
|
+
const { evalCommand } = require('./eval');
|
|
134
|
+
// We can't easily call evalCommand and get the result, so build a lightweight score
|
|
135
|
+
const completionRate = agentEvents.filter(e => e.event === 'crew_end').length /
|
|
136
|
+
Math.max(1, agentEvents.filter(e => e.event === 'crew_start').length);
|
|
137
|
+
const errorRate = errors.length / Math.max(1, lineage.length);
|
|
138
|
+
evalScore = {
|
|
139
|
+
overall: Math.round(Math.max(0, (1 - errorRate) * completionRate * 100)),
|
|
140
|
+
grade: completionRate >= 0.9 && errorRate < 0.1 ? 'A' : completionRate >= 0.7 ? 'B' : 'C',
|
|
141
|
+
dimensions: { completion: Math.round(completionRate * 100), errorRate: Math.round((1 - errorRate) * 100) },
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
catch { }
|
|
145
|
+
// Risk classification
|
|
146
|
+
const riskFactors = [];
|
|
147
|
+
if (llmCalls.length > 0)
|
|
148
|
+
riskFactors.push('Uses AI/LLM for decision-making');
|
|
149
|
+
if (agentEvents.length > 0)
|
|
150
|
+
riskFactors.push('Autonomous agent workflow');
|
|
151
|
+
if (agentTools.length > 0)
|
|
152
|
+
riskFactors.push(`Executes ${agentTools.length} tools autonomously`);
|
|
153
|
+
if (securityFindings.filter(f => f.severity === 'critical').length > 0)
|
|
154
|
+
riskFactors.push('Critical security findings');
|
|
155
|
+
if (errors.length > 0)
|
|
156
|
+
riskFactors.push(`${errors.length} runtime errors`);
|
|
157
|
+
const riskLevel = riskFactors.length >= 3 ? 'high' : riskFactors.length >= 1 ? 'medium' : 'low';
|
|
158
|
+
// Human oversight
|
|
159
|
+
const permissionEvents = agentEvents.filter(e => e.event === 'permission_request' || (e.tool || '').toLowerCase().includes('approval'));
|
|
160
|
+
const escalationEvents = agentEvents.filter(e => e.event?.includes('error') || e.event === 'crew_error');
|
|
161
|
+
const report = {
|
|
162
|
+
meta: {
|
|
163
|
+
generatedAt: new Date().toISOString(),
|
|
164
|
+
trickleVersion: 'CLI 0.1.191',
|
|
165
|
+
framework: [...new Set(agentEvents.map(e => e.framework).filter(Boolean))].join(', ') || 'N/A',
|
|
166
|
+
dataDir: dir,
|
|
167
|
+
},
|
|
168
|
+
riskClassification: { level: riskLevel, factors: riskFactors },
|
|
169
|
+
decisionLineage: lineage,
|
|
170
|
+
dataProcessing: {
|
|
171
|
+
llmProviders: providers, modelsUsed: models,
|
|
172
|
+
totalLlmCalls: llmCalls.length, totalTokens, estimatedCost: Math.round(totalCost * 10000) / 10000,
|
|
173
|
+
toolsUsed: agentTools, mcpToolsUsed: mcpTools,
|
|
174
|
+
dataSourcesAccessed: dataSources.slice(0, 20),
|
|
175
|
+
},
|
|
176
|
+
securityFindings,
|
|
177
|
+
evalScore,
|
|
178
|
+
humanOversight: {
|
|
179
|
+
hasHumanInLoop: permissionEvents.length > 0,
|
|
180
|
+
approvalCheckpoints: permissionEvents.length,
|
|
181
|
+
escalationEvents: escalationEvents.length,
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
// Output
|
|
185
|
+
if (opts.out) {
|
|
186
|
+
fs.writeFileSync(opts.out, JSON.stringify(report, null, 2), 'utf-8');
|
|
187
|
+
console.log(chalk_1.default.green(` Compliance report written to ${opts.out}`));
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (opts.json) {
|
|
191
|
+
console.log(JSON.stringify(report, null, 2));
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
// Pretty print
|
|
195
|
+
console.log('');
|
|
196
|
+
console.log(chalk_1.default.bold(' trickle audit --compliance'));
|
|
197
|
+
console.log(chalk_1.default.gray(' ' + '─'.repeat(60)));
|
|
198
|
+
const riskColor = riskLevel === 'high' ? chalk_1.default.red : riskLevel === 'medium' ? chalk_1.default.yellow : chalk_1.default.green;
|
|
199
|
+
console.log(` Risk: ${riskColor(riskLevel.toUpperCase())} (${riskFactors.length} factors)`);
|
|
200
|
+
for (const f of riskFactors)
|
|
201
|
+
console.log(chalk_1.default.gray(` • ${f}`));
|
|
202
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
203
|
+
console.log(chalk_1.default.bold(' Data Processing'));
|
|
204
|
+
console.log(` LLM providers: ${providers.join(', ') || 'none'}`);
|
|
205
|
+
console.log(` Models: ${models.join(', ') || 'none'}`);
|
|
206
|
+
console.log(` Calls: ${llmCalls.length} Tokens: ${totalTokens} Cost: $${totalCost.toFixed(4)}`);
|
|
207
|
+
console.log(` Tools: ${agentTools.join(', ') || 'none'}`);
|
|
208
|
+
if (mcpTools.length > 0)
|
|
209
|
+
console.log(` MCP tools: ${mcpTools.join(', ')}`);
|
|
210
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
211
|
+
console.log(chalk_1.default.bold(' Decision Lineage'));
|
|
212
|
+
console.log(` ${lineage.length} events recorded`);
|
|
213
|
+
for (const e of lineage.slice(0, 10)) {
|
|
214
|
+
const ts = new Date(e.timestamp).toISOString().substring(11, 23);
|
|
215
|
+
console.log(chalk_1.default.gray(` ${ts}`) + ` ${e.event.padEnd(20)} ${e.description.substring(0, 60)}`);
|
|
216
|
+
}
|
|
217
|
+
if (lineage.length > 10)
|
|
218
|
+
console.log(chalk_1.default.gray(` ... and ${lineage.length - 10} more`));
|
|
219
|
+
if (securityFindings.length > 0) {
|
|
220
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
221
|
+
console.log(chalk_1.default.bold(' Security'));
|
|
222
|
+
const crit = securityFindings.filter(f => f.severity === 'critical').length;
|
|
223
|
+
const warn = securityFindings.filter(f => f.severity === 'warning').length;
|
|
224
|
+
console.log(` ${chalk_1.default.red(String(crit))} critical, ${chalk_1.default.yellow(String(warn))} warnings`);
|
|
225
|
+
}
|
|
226
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
227
|
+
console.log(chalk_1.default.bold(' Human Oversight'));
|
|
228
|
+
console.log(` Human-in-the-loop: ${report.humanOversight.hasHumanInLoop ? chalk_1.default.green('Yes') : chalk_1.default.yellow('No')}`);
|
|
229
|
+
console.log(` Approval checkpoints: ${report.humanOversight.approvalCheckpoints}`);
|
|
230
|
+
console.log(` Escalation events: ${report.humanOversight.escalationEvents}`);
|
|
231
|
+
console.log(chalk_1.default.gray('\n ' + '─'.repeat(60)));
|
|
232
|
+
console.log(chalk_1.default.gray(' Export: trickle audit --compliance --json > audit-report.json'));
|
|
233
|
+
console.log(chalk_1.default.gray(' Export: trickle audit --compliance -o audit-report.json'));
|
|
234
|
+
console.log('');
|
|
235
|
+
}
|
|
236
|
+
function buildAgentDescription(e) {
|
|
237
|
+
const parts = [];
|
|
238
|
+
if (e.chain)
|
|
239
|
+
parts.push(e.chain);
|
|
240
|
+
if (e.tool)
|
|
241
|
+
parts.push(`tool:${e.tool}`);
|
|
242
|
+
if (e.toolInput)
|
|
243
|
+
parts.push(`input:${String(e.toolInput).substring(0, 40)}`);
|
|
244
|
+
if (e.output)
|
|
245
|
+
parts.push(`output:${String(e.output).substring(0, 40)}`);
|
|
246
|
+
if (e.error)
|
|
247
|
+
parts.push(`error:${String(e.error).substring(0, 40)}`);
|
|
248
|
+
if (e.thought)
|
|
249
|
+
parts.push(`thought:${String(e.thought).substring(0, 40)}`);
|
|
250
|
+
return parts.join(' | ') || e.event || '?';
|
|
251
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -396,8 +396,15 @@ program
|
|
|
396
396
|
.option("--json", "Output raw JSON (for CI integration)")
|
|
397
397
|
.option("--fail-on-error", "Exit 1 if any errors are found")
|
|
398
398
|
.option("--fail-on-warning", "Exit 1 if any errors or warnings are found")
|
|
399
|
+
.option("--compliance", "Generate compliance audit report (EU AI Act / Colorado AI Act)")
|
|
400
|
+
.option("-o, --out <file>", "Write compliance report to file")
|
|
399
401
|
.option("--local", "Read from local .trickle/observations.jsonl instead of backend")
|
|
400
402
|
.action(async (opts) => {
|
|
403
|
+
if (opts.compliance) {
|
|
404
|
+
const { generateComplianceReport } = await Promise.resolve().then(() => __importStar(require("./commands/compliance")));
|
|
405
|
+
generateComplianceReport({ json: opts.json, out: opts.out });
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
401
408
|
await (0, audit_1.auditCommand)(opts);
|
|
402
409
|
});
|
|
403
410
|
// trickle capture <method> <url>
|
package/package.json
CHANGED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trickle audit --compliance — Generate compliance audit report.
|
|
3
|
+
*
|
|
4
|
+
* Exports trickle's JSONL data as a structured compliance report for
|
|
5
|
+
* EU AI Act and Colorado AI Act requirements:
|
|
6
|
+
* - Decision lineage (LLM call → tool call → output)
|
|
7
|
+
* - Timestamped event log
|
|
8
|
+
* - Risk classification
|
|
9
|
+
* - Security scan results
|
|
10
|
+
* - Data processing summary
|
|
11
|
+
*
|
|
12
|
+
* Local-first: sensitive audit data never leaves the developer's machine.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
import chalk from 'chalk';
|
|
18
|
+
|
|
19
|
+
function readJsonl(fp: string): any[] {
|
|
20
|
+
if (!fs.existsSync(fp)) return [];
|
|
21
|
+
return fs.readFileSync(fp, 'utf-8').split('\n').filter(Boolean)
|
|
22
|
+
.map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
interface ComplianceReport {
|
|
26
|
+
meta: {
|
|
27
|
+
generatedAt: string;
|
|
28
|
+
trickleVersion: string;
|
|
29
|
+
framework: string;
|
|
30
|
+
dataDir: string;
|
|
31
|
+
};
|
|
32
|
+
riskClassification: {
|
|
33
|
+
level: 'high' | 'medium' | 'low';
|
|
34
|
+
factors: string[];
|
|
35
|
+
};
|
|
36
|
+
decisionLineage: Array<{
|
|
37
|
+
timestamp: number;
|
|
38
|
+
event: string;
|
|
39
|
+
description: string;
|
|
40
|
+
data?: Record<string, unknown>;
|
|
41
|
+
}>;
|
|
42
|
+
dataProcessing: {
|
|
43
|
+
llmProviders: string[];
|
|
44
|
+
modelsUsed: string[];
|
|
45
|
+
totalLlmCalls: number;
|
|
46
|
+
totalTokens: number;
|
|
47
|
+
estimatedCost: number;
|
|
48
|
+
toolsUsed: string[];
|
|
49
|
+
mcpToolsUsed: string[];
|
|
50
|
+
dataSourcesAccessed: string[];
|
|
51
|
+
};
|
|
52
|
+
securityFindings: Array<{
|
|
53
|
+
severity: string;
|
|
54
|
+
category: string;
|
|
55
|
+
message: string;
|
|
56
|
+
evidence: string;
|
|
57
|
+
}>;
|
|
58
|
+
evalScore: {
|
|
59
|
+
overall: number;
|
|
60
|
+
grade: string;
|
|
61
|
+
dimensions: Record<string, number>;
|
|
62
|
+
} | null;
|
|
63
|
+
humanOversight: {
|
|
64
|
+
hasHumanInLoop: boolean;
|
|
65
|
+
approvalCheckpoints: number;
|
|
66
|
+
escalationEvents: number;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function generateComplianceReport(opts: { json?: boolean; out?: string }): void {
|
|
71
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
72
|
+
|
|
73
|
+
if (!fs.existsSync(dir)) {
|
|
74
|
+
console.log(chalk.yellow(' No .trickle/ data. Run trickle first.'));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const llmCalls = readJsonl(path.join(dir, 'llm.jsonl'));
|
|
79
|
+
const agentEvents = readJsonl(path.join(dir, 'agents.jsonl'));
|
|
80
|
+
const mcpCalls = readJsonl(path.join(dir, 'mcp.jsonl'));
|
|
81
|
+
const errors = readJsonl(path.join(dir, 'errors.jsonl'));
|
|
82
|
+
const observations = readJsonl(path.join(dir, 'observations.jsonl'));
|
|
83
|
+
const calltrace = readJsonl(path.join(dir, 'calltrace.jsonl'));
|
|
84
|
+
|
|
85
|
+
// Build decision lineage — chronological event log
|
|
86
|
+
const lineage: ComplianceReport['decisionLineage'] = [];
|
|
87
|
+
|
|
88
|
+
for (const e of agentEvents) {
|
|
89
|
+
lineage.push({
|
|
90
|
+
timestamp: e.timestamp || 0,
|
|
91
|
+
event: `agent:${e.event}`,
|
|
92
|
+
description: buildAgentDescription(e),
|
|
93
|
+
data: { framework: e.framework, tool: e.tool, chain: e.chain },
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
for (const c of llmCalls) {
|
|
98
|
+
lineage.push({
|
|
99
|
+
timestamp: c.timestamp || 0,
|
|
100
|
+
event: 'llm:call',
|
|
101
|
+
description: `${c.provider}/${c.model}: ${(c.inputPreview || '').substring(0, 80)} → ${(c.outputPreview || '').substring(0, 80)}`,
|
|
102
|
+
data: { model: c.model, tokens: c.totalTokens, cost: c.estimatedCostUsd, error: c.error },
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const m of mcpCalls) {
|
|
107
|
+
if (m.tool === '__list_tools') continue;
|
|
108
|
+
lineage.push({
|
|
109
|
+
timestamp: m.timestamp || 0,
|
|
110
|
+
event: `mcp:${m.direction}`,
|
|
111
|
+
description: `${m.tool}(${JSON.stringify(m.args || {}).substring(0, 60)}) → ${(m.resultPreview || '').substring(0, 60)}`,
|
|
112
|
+
data: { tool: m.tool, direction: m.direction, isError: m.isError },
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
lineage.sort((a, b) => a.timestamp - b.timestamp);
|
|
117
|
+
|
|
118
|
+
// Data processing summary
|
|
119
|
+
const providers = [...new Set(llmCalls.map(c => c.provider).filter(Boolean))];
|
|
120
|
+
const models = [...new Set(llmCalls.map(c => `${c.provider}/${c.model}`).filter(Boolean))];
|
|
121
|
+
const totalTokens = llmCalls.reduce((s: number, c: any) => s + (c.totalTokens || 0), 0);
|
|
122
|
+
const totalCost = llmCalls.reduce((s: number, c: any) => s + (c.estimatedCostUsd || 0), 0);
|
|
123
|
+
const agentTools = [...new Set(agentEvents.filter(e => e.tool).map(e => e.tool))];
|
|
124
|
+
const mcpTools = [...new Set(mcpCalls.filter(c => c.tool && c.tool !== '__list_tools').map(c => c.tool))];
|
|
125
|
+
const dataSources = [...new Set(observations.map(o => o.module).filter(Boolean))];
|
|
126
|
+
|
|
127
|
+
// Security scan
|
|
128
|
+
let securityFindings: ComplianceReport['securityFindings'] = [];
|
|
129
|
+
try {
|
|
130
|
+
const { runSecurityScan } = require('./security');
|
|
131
|
+
const origLog = console.log;
|
|
132
|
+
console.log = () => {};
|
|
133
|
+
const result = runSecurityScan({ dir });
|
|
134
|
+
console.log = origLog;
|
|
135
|
+
securityFindings = result.findings.map((f: any) => ({
|
|
136
|
+
severity: f.severity, category: f.category,
|
|
137
|
+
message: f.message, evidence: (f.evidence || '').substring(0, 100),
|
|
138
|
+
}));
|
|
139
|
+
} catch {}
|
|
140
|
+
|
|
141
|
+
// Eval score
|
|
142
|
+
let evalScore: ComplianceReport['evalScore'] = null;
|
|
143
|
+
try {
|
|
144
|
+
const { evalCommand } = require('./eval');
|
|
145
|
+
// We can't easily call evalCommand and get the result, so build a lightweight score
|
|
146
|
+
const completionRate = agentEvents.filter(e => e.event === 'crew_end').length /
|
|
147
|
+
Math.max(1, agentEvents.filter(e => e.event === 'crew_start').length);
|
|
148
|
+
const errorRate = errors.length / Math.max(1, lineage.length);
|
|
149
|
+
evalScore = {
|
|
150
|
+
overall: Math.round(Math.max(0, (1 - errorRate) * completionRate * 100)),
|
|
151
|
+
grade: completionRate >= 0.9 && errorRate < 0.1 ? 'A' : completionRate >= 0.7 ? 'B' : 'C',
|
|
152
|
+
dimensions: { completion: Math.round(completionRate * 100), errorRate: Math.round((1 - errorRate) * 100) },
|
|
153
|
+
};
|
|
154
|
+
} catch {}
|
|
155
|
+
|
|
156
|
+
// Risk classification
|
|
157
|
+
const riskFactors: string[] = [];
|
|
158
|
+
if (llmCalls.length > 0) riskFactors.push('Uses AI/LLM for decision-making');
|
|
159
|
+
if (agentEvents.length > 0) riskFactors.push('Autonomous agent workflow');
|
|
160
|
+
if (agentTools.length > 0) riskFactors.push(`Executes ${agentTools.length} tools autonomously`);
|
|
161
|
+
if (securityFindings.filter(f => f.severity === 'critical').length > 0) riskFactors.push('Critical security findings');
|
|
162
|
+
if (errors.length > 0) riskFactors.push(`${errors.length} runtime errors`);
|
|
163
|
+
const riskLevel: 'high' | 'medium' | 'low' = riskFactors.length >= 3 ? 'high' : riskFactors.length >= 1 ? 'medium' : 'low';
|
|
164
|
+
|
|
165
|
+
// Human oversight
|
|
166
|
+
const permissionEvents = agentEvents.filter(e =>
|
|
167
|
+
e.event === 'permission_request' || (e.tool || '').toLowerCase().includes('approval'));
|
|
168
|
+
const escalationEvents = agentEvents.filter(e =>
|
|
169
|
+
e.event?.includes('error') || e.event === 'crew_error');
|
|
170
|
+
|
|
171
|
+
const report: ComplianceReport = {
|
|
172
|
+
meta: {
|
|
173
|
+
generatedAt: new Date().toISOString(),
|
|
174
|
+
trickleVersion: 'CLI 0.1.191',
|
|
175
|
+
framework: [...new Set(agentEvents.map(e => e.framework).filter(Boolean))].join(', ') || 'N/A',
|
|
176
|
+
dataDir: dir,
|
|
177
|
+
},
|
|
178
|
+
riskClassification: { level: riskLevel, factors: riskFactors },
|
|
179
|
+
decisionLineage: lineage,
|
|
180
|
+
dataProcessing: {
|
|
181
|
+
llmProviders: providers, modelsUsed: models,
|
|
182
|
+
totalLlmCalls: llmCalls.length, totalTokens, estimatedCost: Math.round(totalCost * 10000) / 10000,
|
|
183
|
+
toolsUsed: agentTools, mcpToolsUsed: mcpTools,
|
|
184
|
+
dataSourcesAccessed: dataSources.slice(0, 20),
|
|
185
|
+
},
|
|
186
|
+
securityFindings,
|
|
187
|
+
evalScore,
|
|
188
|
+
humanOversight: {
|
|
189
|
+
hasHumanInLoop: permissionEvents.length > 0,
|
|
190
|
+
approvalCheckpoints: permissionEvents.length,
|
|
191
|
+
escalationEvents: escalationEvents.length,
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
// Output
|
|
196
|
+
if (opts.out) {
|
|
197
|
+
fs.writeFileSync(opts.out, JSON.stringify(report, null, 2), 'utf-8');
|
|
198
|
+
console.log(chalk.green(` Compliance report written to ${opts.out}`));
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (opts.json) {
|
|
203
|
+
console.log(JSON.stringify(report, null, 2));
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Pretty print
|
|
208
|
+
console.log('');
|
|
209
|
+
console.log(chalk.bold(' trickle audit --compliance'));
|
|
210
|
+
console.log(chalk.gray(' ' + '─'.repeat(60)));
|
|
211
|
+
|
|
212
|
+
const riskColor = riskLevel === 'high' ? chalk.red : riskLevel === 'medium' ? chalk.yellow : chalk.green;
|
|
213
|
+
console.log(` Risk: ${riskColor(riskLevel.toUpperCase())} (${riskFactors.length} factors)`);
|
|
214
|
+
for (const f of riskFactors) console.log(chalk.gray(` • ${f}`));
|
|
215
|
+
|
|
216
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
217
|
+
console.log(chalk.bold(' Data Processing'));
|
|
218
|
+
console.log(` LLM providers: ${providers.join(', ') || 'none'}`);
|
|
219
|
+
console.log(` Models: ${models.join(', ') || 'none'}`);
|
|
220
|
+
console.log(` Calls: ${llmCalls.length} Tokens: ${totalTokens} Cost: $${totalCost.toFixed(4)}`);
|
|
221
|
+
console.log(` Tools: ${agentTools.join(', ') || 'none'}`);
|
|
222
|
+
if (mcpTools.length > 0) console.log(` MCP tools: ${mcpTools.join(', ')}`);
|
|
223
|
+
|
|
224
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
225
|
+
console.log(chalk.bold(' Decision Lineage'));
|
|
226
|
+
console.log(` ${lineage.length} events recorded`);
|
|
227
|
+
for (const e of lineage.slice(0, 10)) {
|
|
228
|
+
const ts = new Date(e.timestamp).toISOString().substring(11, 23);
|
|
229
|
+
console.log(chalk.gray(` ${ts}`) + ` ${e.event.padEnd(20)} ${e.description.substring(0, 60)}`);
|
|
230
|
+
}
|
|
231
|
+
if (lineage.length > 10) console.log(chalk.gray(` ... and ${lineage.length - 10} more`));
|
|
232
|
+
|
|
233
|
+
if (securityFindings.length > 0) {
|
|
234
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
235
|
+
console.log(chalk.bold(' Security'));
|
|
236
|
+
const crit = securityFindings.filter(f => f.severity === 'critical').length;
|
|
237
|
+
const warn = securityFindings.filter(f => f.severity === 'warning').length;
|
|
238
|
+
console.log(` ${chalk.red(String(crit))} critical, ${chalk.yellow(String(warn))} warnings`);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
242
|
+
console.log(chalk.bold(' Human Oversight'));
|
|
243
|
+
console.log(` Human-in-the-loop: ${report.humanOversight.hasHumanInLoop ? chalk.green('Yes') : chalk.yellow('No')}`);
|
|
244
|
+
console.log(` Approval checkpoints: ${report.humanOversight.approvalCheckpoints}`);
|
|
245
|
+
console.log(` Escalation events: ${report.humanOversight.escalationEvents}`);
|
|
246
|
+
|
|
247
|
+
console.log(chalk.gray('\n ' + '─'.repeat(60)));
|
|
248
|
+
console.log(chalk.gray(' Export: trickle audit --compliance --json > audit-report.json'));
|
|
249
|
+
console.log(chalk.gray(' Export: trickle audit --compliance -o audit-report.json'));
|
|
250
|
+
console.log('');
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function buildAgentDescription(e: any): string {
|
|
254
|
+
const parts: string[] = [];
|
|
255
|
+
if (e.chain) parts.push(e.chain);
|
|
256
|
+
if (e.tool) parts.push(`tool:${e.tool}`);
|
|
257
|
+
if (e.toolInput) parts.push(`input:${String(e.toolInput).substring(0, 40)}`);
|
|
258
|
+
if (e.output) parts.push(`output:${String(e.output).substring(0, 40)}`);
|
|
259
|
+
if (e.error) parts.push(`error:${String(e.error).substring(0, 40)}`);
|
|
260
|
+
if (e.thought) parts.push(`thought:${String(e.thought).substring(0, 40)}`);
|
|
261
|
+
return parts.join(' | ') || e.event || '?';
|
|
262
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -384,8 +384,15 @@ program
|
|
|
384
384
|
.option("--json", "Output raw JSON (for CI integration)")
|
|
385
385
|
.option("--fail-on-error", "Exit 1 if any errors are found")
|
|
386
386
|
.option("--fail-on-warning", "Exit 1 if any errors or warnings are found")
|
|
387
|
+
.option("--compliance", "Generate compliance audit report (EU AI Act / Colorado AI Act)")
|
|
388
|
+
.option("-o, --out <file>", "Write compliance report to file")
|
|
387
389
|
.option("--local", "Read from local .trickle/observations.jsonl instead of backend")
|
|
388
390
|
.action(async (opts) => {
|
|
391
|
+
if (opts.compliance) {
|
|
392
|
+
const { generateComplianceReport } = await import("./commands/compliance");
|
|
393
|
+
generateComplianceReport({ json: opts.json, out: opts.out });
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
389
396
|
await auditCommand(opts);
|
|
390
397
|
});
|
|
391
398
|
|