watchmyagents 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -9
- package/SECURITY.md +9 -3
- package/package.json +10 -4
- package/scripts/anonymize.js +121 -0
- package/scripts/fetch-anthropic.js +10 -0
- package/scripts/inspect.js +14 -4
- package/scripts/shield.js +164 -16
- package/scripts/upload-fortress.js +222 -0
- package/src/anonymizer.js +206 -0
- package/src/fortress/url.js +59 -0
- package/src/logger.js +11 -2
- package/src/shield/policy.js +50 -7
- package/src/shield/sources/fortress.js +203 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// wma-upload-fortress — anonymize local Watch NDJSON and POST signals to
|
|
3
|
+
// the Fortress ingest-signals Edge Function.
|
|
4
|
+
//
|
|
5
|
+
// Composable with the rest of the SDK:
|
|
6
|
+
// wma-fetch → ./watchmyagents-logs/<agent_id>/<date>.ndjson (local capture)
|
|
7
|
+
// wma-anonymize → signals payload (Modèle C: no raw content)
|
|
8
|
+
// wma-upload-fortress → POST signals to https://<project>.supabase.co/functions/v1/ingest-signals
|
|
9
|
+
//
|
|
10
|
+
// Usage:
|
|
11
|
+
// wma-upload-fortress --agent-id agent_xxx \
|
|
12
|
+
// [--log-dir ./watchmyagents-logs] \
|
|
13
|
+
// [--fortress-url https://<project>.supabase.co/functions/v1/ingest-signals] \
|
|
14
|
+
// [--api-key wma_...] \
|
|
15
|
+
// [--salt <hex>] \
|
|
16
|
+
// [--display-name "My agent"] \
|
|
17
|
+
// [--dry-run]
|
|
18
|
+
//
|
|
19
|
+
// Env vars (preferred over CLI flags):
|
|
20
|
+
// WMA_API_KEY — the wma_xxx key from the Fortress dashboard
|
|
21
|
+
// WMA_FORTRESS_URL — full URL to the ingest-signals endpoint
|
|
22
|
+
// WMA_SIGNALS_SALT — per-customer hex salt for IoC hashing
|
|
23
|
+
// (must be stable across runs)
|
|
24
|
+
|
|
25
|
+
import { request as httpsRequest } from 'node:https';
|
|
26
|
+
import { URL } from 'node:url';
|
|
27
|
+
import { readdir, stat } from 'node:fs/promises';
|
|
28
|
+
import { join, resolve } from 'node:path';
|
|
29
|
+
import { createReadStream } from 'node:fs';
|
|
30
|
+
import { createInterface } from 'node:readline';
|
|
31
|
+
import { SignalsAggregator } from '../src/anonymizer.js';
|
|
32
|
+
import { resolveFortressBase, fortressEndpoint } from '../src/fortress/url.js';
|
|
33
|
+
|
|
34
|
+
function parseArgs(argv) {
|
|
35
|
+
const out = {};
|
|
36
|
+
for (let i = 0; i < argv.length; i++) {
|
|
37
|
+
const a = argv[i];
|
|
38
|
+
if (a.startsWith('--')) {
|
|
39
|
+
const k = a.slice(2);
|
|
40
|
+
const n = argv[i + 1];
|
|
41
|
+
if (n == null || n.startsWith('--')) out[k] = true;
|
|
42
|
+
else { out[k] = n; i++; }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function die(msg, code = 1) { process.stderr.write(`${msg}\n`); process.exit(code); }
|
|
49
|
+
function info(msg) { process.stdout.write(`[wma-upload-fortress] ${msg}\n`); }
|
|
50
|
+
function warn(msg) { process.stderr.write(`[wma-upload-fortress] ⚠️ ${msg}\n`); }
|
|
51
|
+
|
|
52
|
+
async function collectFiles(p) {
|
|
53
|
+
const s = await stat(p).catch(() => null);
|
|
54
|
+
if (!s) return [];
|
|
55
|
+
if (s.isFile()) return p.endsWith('.ndjson') && !p.includes('raw-') ? [p] : [];
|
|
56
|
+
const out = [];
|
|
57
|
+
for (const name of await readdir(p)) {
|
|
58
|
+
out.push(...(await collectFiles(join(p, name))));
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function postJson(url, headers, body) {
|
|
64
|
+
return new Promise((resolveReq, rejectReq) => {
|
|
65
|
+
const u = new URL(url);
|
|
66
|
+
if (u.protocol !== 'https:') {
|
|
67
|
+
return rejectReq(new Error(`refusing non-https fortress URL: ${url}`));
|
|
68
|
+
}
|
|
69
|
+
const data = Buffer.from(body);
|
|
70
|
+
const req = httpsRequest(
|
|
71
|
+
{
|
|
72
|
+
method: 'POST',
|
|
73
|
+
hostname: u.hostname,
|
|
74
|
+
port: u.port || 443,
|
|
75
|
+
path: u.pathname + u.search,
|
|
76
|
+
headers: {
|
|
77
|
+
...headers,
|
|
78
|
+
'content-type': 'application/json',
|
|
79
|
+
'content-length': data.length,
|
|
80
|
+
},
|
|
81
|
+
rejectUnauthorized: true,
|
|
82
|
+
},
|
|
83
|
+
(res) => {
|
|
84
|
+
const chunks = [];
|
|
85
|
+
res.on('data', (c) => chunks.push(c));
|
|
86
|
+
res.on('end', () => {
|
|
87
|
+
const raw = Buffer.concat(chunks).toString('utf8');
|
|
88
|
+
let parsed = null;
|
|
89
|
+
try { parsed = JSON.parse(raw); } catch { /* keep raw */ }
|
|
90
|
+
resolveReq({ status: res.statusCode || 0, body: parsed ?? raw });
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
);
|
|
94
|
+
req.on('error', rejectReq);
|
|
95
|
+
req.write(data);
|
|
96
|
+
req.end();
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function main() {
|
|
101
|
+
const args = parseArgs(process.argv.slice(2));
|
|
102
|
+
|
|
103
|
+
const agentId = args['agent-id'];
|
|
104
|
+
const logDir = resolve(args['log-dir'] || './watchmyagents-logs');
|
|
105
|
+
const apiKey = args['api-key'] || process.env.WMA_API_KEY;
|
|
106
|
+
const salt = args.salt || process.env.WMA_SIGNALS_SALT;
|
|
107
|
+
const displayName = args['display-name'] || agentId;
|
|
108
|
+
const dryRun = !!args['dry-run'];
|
|
109
|
+
|
|
110
|
+
// Resolve Fortress base URL. Accepts:
|
|
111
|
+
// --fortress-base-url <base> (preferred CLI)
|
|
112
|
+
// --fortress-url <full ingest-signals> (legacy CLI)
|
|
113
|
+
// WMA_FORTRESS_BASE_URL env (preferred env)
|
|
114
|
+
// WMA_FORTRESS_URL env (legacy env, points at ingest-signals)
|
|
115
|
+
const fortressBase = resolveFortressBase({
|
|
116
|
+
explicitBase: args['fortress-base-url'],
|
|
117
|
+
explicitUrl: args['fortress-url'],
|
|
118
|
+
});
|
|
119
|
+
const fortressUrl = fortressBase ? fortressEndpoint(fortressBase, 'ingest-signals') : null;
|
|
120
|
+
|
|
121
|
+
// Validation
|
|
122
|
+
if (!agentId) die('error: --agent-id required (Anthropic agent_id, e.g. agent_01XaN...)');
|
|
123
|
+
// Strict alphanumeric to prevent path traversal in collectFiles below
|
|
124
|
+
// (--agent-id ends up as a filesystem path segment).
|
|
125
|
+
if (!/^agent_[a-zA-Z0-9]+$/.test(agentId)) {
|
|
126
|
+
die(`error: --agent-id has invalid format (expected "agent_" + alphanumeric, got "${agentId}")`);
|
|
127
|
+
}
|
|
128
|
+
if (!dryRun && !fortressUrl) {
|
|
129
|
+
die('error: --fortress-url or WMA_FORTRESS_URL required (full URL to /functions/v1/ingest-signals).\n' +
|
|
130
|
+
' Use --dry-run to print the payload without uploading.');
|
|
131
|
+
}
|
|
132
|
+
if (!dryRun && !apiKey) {
|
|
133
|
+
die('error: --api-key or WMA_API_KEY required.\n' +
|
|
134
|
+
' Get one from your Fortress dashboard → Settings → API Keys.');
|
|
135
|
+
}
|
|
136
|
+
if (!dryRun && apiKey && !/^wma_[a-f0-9]{32}$/i.test(apiKey)) {
|
|
137
|
+
warn(`API key format looks unusual (expected "wma_<32hex>", got "${apiKey.slice(0, 8)}…").`);
|
|
138
|
+
}
|
|
139
|
+
if (!salt) {
|
|
140
|
+
die('error: --salt or WMA_SIGNALS_SALT required (per-customer hex secret for hashing IoCs).\n' +
|
|
141
|
+
' Generate once with: node -e "console.log(require(\'crypto\').randomBytes(16).toString(\'hex\'))"\n' +
|
|
142
|
+
' Store stably in .env.local.');
|
|
143
|
+
}
|
|
144
|
+
if (salt.length < 16) die('error: salt too short (need ≥16 hex chars)');
|
|
145
|
+
|
|
146
|
+
// Warn about CLI-passed secrets
|
|
147
|
+
if (args['api-key']) {
|
|
148
|
+
warn('--api-key on the command line is visible in shell history and process list.\n' +
|
|
149
|
+
' Prefer: export WMA_API_KEY=...');
|
|
150
|
+
}
|
|
151
|
+
if (args.salt) {
|
|
152
|
+
warn('--salt on the command line is visible in shell history.\n' +
|
|
153
|
+
' Prefer: export WMA_SIGNALS_SALT=...');
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Discover the agent's NDJSON files
|
|
157
|
+
const agentDir = join(logDir, agentId);
|
|
158
|
+
const files = await collectFiles(agentDir);
|
|
159
|
+
if (files.length === 0) {
|
|
160
|
+
die(`error: no .ndjson files found under ${agentDir}. Run wma-fetch first?`);
|
|
161
|
+
}
|
|
162
|
+
info(`scanning ${files.length} ndjson file(s) under ${agentDir}`);
|
|
163
|
+
|
|
164
|
+
// Aggregate into a single signals payload
|
|
165
|
+
const agg = new SignalsAggregator({ salt });
|
|
166
|
+
for (const f of files) {
|
|
167
|
+
const stream = createReadStream(f, { encoding: 'utf8' });
|
|
168
|
+
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
169
|
+
for await (const line of rl) {
|
|
170
|
+
if (!line.trim()) continue;
|
|
171
|
+
let e; try { e = JSON.parse(line); } catch { continue; }
|
|
172
|
+
agg.add(e);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const signals = agg.finalize();
|
|
176
|
+
if (!signals.window_start || !signals.window_end) {
|
|
177
|
+
die('error: no entries had timestamps — nothing to upload');
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const body = {
|
|
181
|
+
anthropic_agent_id: agentId,
|
|
182
|
+
display_name: displayName,
|
|
183
|
+
window_start: signals.window_start,
|
|
184
|
+
window_end: signals.window_end,
|
|
185
|
+
payload: signals.payload,
|
|
186
|
+
};
|
|
187
|
+
const bodyJson = JSON.stringify(body);
|
|
188
|
+
|
|
189
|
+
info(`payload built: ${signals._meta.entries_processed} entries → ${bodyJson.length} bytes`);
|
|
190
|
+
info(`window: ${signals.window_start} → ${signals.window_end}`);
|
|
191
|
+
info(`ioc_hashes: ${signals.payload.ioc_hashes.length}, tool_counts: ${Object.keys(signals.payload.tool_counts).length}`);
|
|
192
|
+
|
|
193
|
+
if (dryRun) {
|
|
194
|
+
info('--dry-run: payload that WOULD be POSTed:');
|
|
195
|
+
process.stdout.write(JSON.stringify(body, null, 2) + '\n');
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// POST it
|
|
200
|
+
info(`POST ${fortressUrl}`);
|
|
201
|
+
const { status, body: respBody } = await postJson(
|
|
202
|
+
fortressUrl,
|
|
203
|
+
{ authorization: `Bearer ${apiKey}` },
|
|
204
|
+
bodyJson
|
|
205
|
+
);
|
|
206
|
+
|
|
207
|
+
if (status >= 200 && status < 300) {
|
|
208
|
+
info(`✅ HTTP ${status}`);
|
|
209
|
+
if (typeof respBody === 'object' && respBody.signal_id) {
|
|
210
|
+
info(`signal_id: ${respBody.signal_id}`);
|
|
211
|
+
info(`agent_id: ${respBody.agent_id}`);
|
|
212
|
+
if (respBody.registered_new_agent) info('🆕 agent was auto-registered on this upload');
|
|
213
|
+
} else {
|
|
214
|
+
info(`response: ${typeof respBody === 'string' ? respBody.slice(0, 300) : JSON.stringify(respBody).slice(0, 300)}`);
|
|
215
|
+
}
|
|
216
|
+
} else {
|
|
217
|
+
const msg = typeof respBody === 'object' ? JSON.stringify(respBody) : String(respBody).slice(0, 500);
|
|
218
|
+
die(`error: upload failed (HTTP ${status}): ${msg}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
main().catch((e) => { process.stderr.write(`error: ${e.stack || e.message}\n`); process.exit(1); });
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
2
|
+
// Anonymizer — strip raw payloads, produce signals safe for Fortress
|
|
3
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
4
|
+
// Reads a Watch NDJSON file (the full local log) and produces an
|
|
5
|
+
// anonymized signals payload — the shape Fortress's `signals` table
|
|
6
|
+
// expects. The output contains ONLY:
|
|
7
|
+
//
|
|
8
|
+
// - counts (action_type, tool_name)
|
|
9
|
+
// - latencies (p50, p95, max) per tool
|
|
10
|
+
// - error rates per tool
|
|
11
|
+
// - salted SHA-256 hashes of IoCs (URLs, commands, queries)
|
|
12
|
+
// - top action_type sequences (Markov pairs)
|
|
13
|
+
// - stop_reason type counts (NOT the message text)
|
|
14
|
+
// - tokens_total
|
|
15
|
+
//
|
|
16
|
+
// What it NEVER outputs:
|
|
17
|
+
// - input.content (prompts)
|
|
18
|
+
// - output.content (agent text)
|
|
19
|
+
// - raw URLs / commands / queries
|
|
20
|
+
// - error messages
|
|
21
|
+
// - readable session_id (hashed)
|
|
22
|
+
// - readable agent_id (hashed)
|
|
23
|
+
// - PII of any kind
|
|
24
|
+
//
|
|
25
|
+
// This is the single bottleneck between Watch (local) and Fortress (cloud).
|
|
26
|
+
// Every byte that crosses to the cloud passes through this module.
|
|
27
|
+
|
|
28
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
29
|
+
import { createReadStream } from 'node:fs';
|
|
30
|
+
import { createInterface } from 'node:readline';
|
|
31
|
+
|
|
32
|
+
// ── Configuration ────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
// Fields that may contain raw data — we extract a hash, never the raw value.
|
|
35
|
+
const HASHABLE_INPUT_FIELDS = ['url', 'query', 'command', 'path', 'file_path'];
|
|
36
|
+
|
|
37
|
+
// Tool types whose inputs we want to hash for IoC tracking
|
|
38
|
+
const TOOL_ACTIONS = new Set(['tool_use', 'mcp_tool_use', 'custom_tool_use']);
|
|
39
|
+
|
|
40
|
+
// ── Hash helpers ─────────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Salted SHA-256 hash. The salt is per-customer (passed in) so the same URL
|
|
44
|
+
* at customer A produces a different hash than at customer B by default —
|
|
45
|
+
* but if a global salt is used, identical IoCs across customers produce
|
|
46
|
+
* identical hashes (the antivirus model for L4 cross-customer intel).
|
|
47
|
+
*/
|
|
48
|
+
export function hashWithSalt(value, salt) {
|
|
49
|
+
if (value == null) return null;
|
|
50
|
+
const s = typeof value === 'string' ? value : JSON.stringify(value);
|
|
51
|
+
return 'sha256:' + createHash('sha256').update(salt).update(s).digest('hex').slice(0, 32);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Generate a customer salt (if none provided)
|
|
55
|
+
export function generateSalt() {
|
|
56
|
+
return randomBytes(16).toString('hex');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// ── Single-entry extractor: what hashable IoCs are in this entry? ────────
|
|
60
|
+
|
|
61
|
+
function extractIocs(entry, salt) {
|
|
62
|
+
const out = [];
|
|
63
|
+
if (!entry.input || typeof entry.input !== 'object') return out;
|
|
64
|
+
for (const field of HASHABLE_INPUT_FIELDS) {
|
|
65
|
+
const v = entry.input[field];
|
|
66
|
+
if (typeof v === 'string' && v.length > 0) {
|
|
67
|
+
out.push(hashWithSalt(v, salt));
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return out;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ── Aggregator: walks the NDJSON stream and builds the signals payload ──
|
|
74
|
+
|
|
75
|
+
export class SignalsAggregator {
|
|
76
|
+
constructor({ salt } = {}) {
|
|
77
|
+
if (!salt) throw new Error('SignalsAggregator requires a salt');
|
|
78
|
+
this.salt = salt;
|
|
79
|
+
this.counts = Object.create(null); // action_type → count
|
|
80
|
+
this.toolCounts = Object.create(null); // tool_name → count
|
|
81
|
+
this.toolErrors = Object.create(null); // tool_name → error count
|
|
82
|
+
this.toolLatencies = Object.create(null); // tool_name → number[]
|
|
83
|
+
this.iocHashes = new Set(); // unique IoC hashes
|
|
84
|
+
this.sequences = Object.create(null); // "A → B" → count
|
|
85
|
+
this.stopReasons = Object.create(null); // stop_reason.type → count
|
|
86
|
+
this.tokensTotal = 0;
|
|
87
|
+
this.windowStart = null;
|
|
88
|
+
this.windowEnd = null;
|
|
89
|
+
this.entryCount = 0;
|
|
90
|
+
this._prevActionType = null;
|
|
91
|
+
this._prevSessionId = null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
add(entry) {
|
|
95
|
+
if (!entry) return;
|
|
96
|
+
this.entryCount++;
|
|
97
|
+
|
|
98
|
+
// Track window bounds
|
|
99
|
+
const ts = entry.timestamp || '';
|
|
100
|
+
if (ts) {
|
|
101
|
+
if (!this.windowStart || ts < this.windowStart) this.windowStart = ts;
|
|
102
|
+
if (!this.windowEnd || ts > this.windowEnd) this.windowEnd = ts;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Counts
|
|
106
|
+
const at = entry.action_type || 'unknown';
|
|
107
|
+
this.counts[at] = (this.counts[at] || 0) + 1;
|
|
108
|
+
|
|
109
|
+
// Sequence tracking (only within the same session)
|
|
110
|
+
if (this._prevActionType && entry.session_id === this._prevSessionId
|
|
111
|
+
&& at !== 'session_end' && this._prevActionType !== 'session_end') {
|
|
112
|
+
const seqKey = `${this._prevActionType} → ${at}`;
|
|
113
|
+
this.sequences[seqKey] = (this.sequences[seqKey] || 0) + 1;
|
|
114
|
+
}
|
|
115
|
+
this._prevActionType = at;
|
|
116
|
+
this._prevSessionId = entry.session_id || null;
|
|
117
|
+
|
|
118
|
+
// Tools
|
|
119
|
+
if (entry.tool_name && TOOL_ACTIONS.has(at)) {
|
|
120
|
+
this.toolCounts[entry.tool_name] = (this.toolCounts[entry.tool_name] || 0) + 1;
|
|
121
|
+
if (entry.status === 'error') {
|
|
122
|
+
this.toolErrors[entry.tool_name] = (this.toolErrors[entry.tool_name] || 0) + 1;
|
|
123
|
+
}
|
|
124
|
+
if (typeof entry.duration_ms === 'number') {
|
|
125
|
+
if (!this.toolLatencies[entry.tool_name]) this.toolLatencies[entry.tool_name] = [];
|
|
126
|
+
this.toolLatencies[entry.tool_name].push(entry.duration_ms);
|
|
127
|
+
}
|
|
128
|
+
// Extract & hash IoCs from this tool's input
|
|
129
|
+
for (const h of extractIocs(entry, this.salt)) this.iocHashes.add(h);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Tokens
|
|
133
|
+
if (typeof entry.tokens_used === 'number') this.tokensTotal += entry.tokens_used;
|
|
134
|
+
|
|
135
|
+
// Stop reasons (state_transition entries carry these)
|
|
136
|
+
const stopType = entry.output?.stop_reason?.type;
|
|
137
|
+
if (typeof stopType === 'string') {
|
|
138
|
+
this.stopReasons[stopType] = (this.stopReasons[stopType] || 0) + 1;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Compute p50/p95/max for an array of durations
|
|
143
|
+
_percentiles(arr) {
|
|
144
|
+
if (arr.length === 0) return null;
|
|
145
|
+
const sorted = [...arr].sort((a, b) => a - b);
|
|
146
|
+
const at = (p) => sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
|
|
147
|
+
return { p50: at(50), p95: at(95), max: sorted[sorted.length - 1], n: sorted.length };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
finalize() {
|
|
151
|
+
// Latencies aggregated
|
|
152
|
+
const latencies_p50_ms = {};
|
|
153
|
+
const latencies_p95_ms = {};
|
|
154
|
+
const error_rate_by_tool = {};
|
|
155
|
+
for (const [tool, durations] of Object.entries(this.toolLatencies)) {
|
|
156
|
+
const p = this._percentiles(durations);
|
|
157
|
+
if (p) {
|
|
158
|
+
latencies_p50_ms[tool] = p.p50;
|
|
159
|
+
latencies_p95_ms[tool] = p.p95;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
for (const tool of Object.keys(this.toolCounts)) {
|
|
163
|
+
const errs = this.toolErrors[tool] || 0;
|
|
164
|
+
error_rate_by_tool[tool] = +(errs / this.toolCounts[tool]).toFixed(4);
|
|
165
|
+
}
|
|
166
|
+
// Top-10 sequences
|
|
167
|
+
const sequencesTop = Object.entries(this.sequences)
|
|
168
|
+
.sort((a, b) => b[1] - a[1])
|
|
169
|
+
.slice(0, 10)
|
|
170
|
+
.map(([pattern, count]) => ({ pattern, count }));
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
window_start: this.windowStart,
|
|
174
|
+
window_end: this.windowEnd,
|
|
175
|
+
payload: {
|
|
176
|
+
counts: this.counts,
|
|
177
|
+
tool_counts: this.toolCounts,
|
|
178
|
+
latencies_p50_ms,
|
|
179
|
+
latencies_p95_ms,
|
|
180
|
+
error_rate_by_tool,
|
|
181
|
+
ioc_hashes: [...this.iocHashes],
|
|
182
|
+
sequences_top10: sequencesTop,
|
|
183
|
+
stop_reasons: this.stopReasons,
|
|
184
|
+
tokens_total: this.tokensTotal,
|
|
185
|
+
},
|
|
186
|
+
_meta: {
|
|
187
|
+
entries_processed: this.entryCount,
|
|
188
|
+
},
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ── Streaming convenience: anonymize a whole NDJSON file/dir ────────────
|
|
194
|
+
|
|
195
|
+
export async function anonymizeFile(filePath, { salt } = {}) {
|
|
196
|
+
if (!salt) throw new Error('anonymizeFile requires a salt');
|
|
197
|
+
const agg = new SignalsAggregator({ salt });
|
|
198
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
199
|
+
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
200
|
+
for await (const line of rl) {
|
|
201
|
+
if (!line.trim()) continue;
|
|
202
|
+
let e; try { e = JSON.parse(line); } catch { continue; }
|
|
203
|
+
agg.add(e);
|
|
204
|
+
}
|
|
205
|
+
return agg.finalize();
|
|
206
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
2
|
+
// Fortress URL resolution — shared across upload-fortress, shield, etc.
|
|
3
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
4
|
+
// The user sets ONE of:
|
|
5
|
+
//
|
|
6
|
+
// WMA_FORTRESS_BASE_URL=https://<project>.supabase.co/functions/v1
|
|
7
|
+
// → preferred. Each tool appends its endpoint (/ingest-signals,
|
|
8
|
+
// /get-policies, /ingest-decisions).
|
|
9
|
+
//
|
|
10
|
+
// WMA_FORTRESS_URL=https://<project>.supabase.co/functions/v1/ingest-signals
|
|
11
|
+
// → legacy (v0.5.0 era). The base URL is derived by stripping the
|
|
12
|
+
// last path segment, so other endpoints can be constructed.
|
|
13
|
+
//
|
|
14
|
+
// Either way, callers receive a `base` they append `/<endpoint>` to.
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Resolve the Fortress base URL from env / args.
|
|
18
|
+
* @param {object} opts - { explicitUrl, explicitBase, env }
|
|
19
|
+
* @returns {string|null} base URL like https://x.supabase.co/functions/v1
|
|
20
|
+
* (no trailing slash), or null if not configured.
|
|
21
|
+
*/
|
|
22
|
+
export function resolveFortressBase({ explicitUrl, explicitBase, env = process.env } = {}) {
|
|
23
|
+
// 1. Explicit base URL from CLI
|
|
24
|
+
if (explicitBase) return stripTrailingSlash(explicitBase);
|
|
25
|
+
|
|
26
|
+
// 2. Env: WMA_FORTRESS_BASE_URL (preferred)
|
|
27
|
+
if (env.WMA_FORTRESS_BASE_URL) return stripTrailingSlash(env.WMA_FORTRESS_BASE_URL);
|
|
28
|
+
|
|
29
|
+
// 3. Legacy: WMA_FORTRESS_URL (full path to ingest-signals)
|
|
30
|
+
const legacy = explicitUrl || env.WMA_FORTRESS_URL;
|
|
31
|
+
if (legacy) {
|
|
32
|
+
// Strip last path segment to get the base
|
|
33
|
+
try {
|
|
34
|
+
const u = new URL(legacy);
|
|
35
|
+
const parts = u.pathname.split('/').filter(Boolean);
|
|
36
|
+
if (parts.length > 0) parts.pop();
|
|
37
|
+
u.pathname = '/' + parts.join('/');
|
|
38
|
+
return stripTrailingSlash(u.toString());
|
|
39
|
+
} catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function stripTrailingSlash(s) {
|
|
48
|
+
return s.endsWith('/') ? s.slice(0, -1) : s;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a full endpoint URL given a base + endpoint name.
|
|
53
|
+
* @param {string} base - e.g. https://x.supabase.co/functions/v1
|
|
54
|
+
* @param {string} endpoint - e.g. "ingest-signals", "get-policies"
|
|
55
|
+
*/
|
|
56
|
+
export function fortressEndpoint(base, endpoint) {
|
|
57
|
+
if (!base) throw new Error('Fortress base URL not configured');
|
|
58
|
+
return `${base}/${endpoint}`;
|
|
59
|
+
}
|
package/src/logger.js
CHANGED
|
@@ -12,11 +12,17 @@ const EXPORT_FIELDS = [
|
|
|
12
12
|
];
|
|
13
13
|
|
|
14
14
|
export class Logger {
|
|
15
|
-
|
|
15
|
+
// `silent` : don't print log errors to stderr (default: true — quiet operation)
|
|
16
|
+
// `bestEffort` : SWALLOW write failures (default: false — fail loud).
|
|
17
|
+
// Audit-grade default: refuse to silently lose events. Disk
|
|
18
|
+
// full / EACCES / EINVAL must propagate so callers know.
|
|
19
|
+
// Opt into bestEffort=true only for non-critical paths.
|
|
20
|
+
constructor({ logDir, agentId, sessionId, silent, bestEffort } = {}) {
|
|
16
21
|
this.logDir = logDir;
|
|
17
22
|
this.agentId = agentId;
|
|
18
23
|
this.sessionId = sessionId || randomUUID();
|
|
19
24
|
this.silent = silent !== false;
|
|
25
|
+
this.bestEffort = bestEffort === true;
|
|
20
26
|
this.sequence = 0;
|
|
21
27
|
this.currentDay = null;
|
|
22
28
|
this.currentPath = null;
|
|
@@ -63,7 +69,10 @@ export class Logger {
|
|
|
63
69
|
await appendFile(path, JSON.stringify(full) + '\n', { encoding: 'utf8', mode: 0o600 });
|
|
64
70
|
this.count++;
|
|
65
71
|
} catch (err) {
|
|
66
|
-
if (!this.silent) process.stderr.write(`[wma] log error: ${err.message}\n`);
|
|
72
|
+
if (!this.silent) process.stderr.write(`[wma] log write error: ${err.message}\n`);
|
|
73
|
+
// Audit-grade default: fail loud so callers know events are being lost.
|
|
74
|
+
// Disk full, EACCES, EINVAL etc. should NOT be silently swallowed.
|
|
75
|
+
if (!this.bestEffort) throw err;
|
|
67
76
|
}
|
|
68
77
|
return full;
|
|
69
78
|
}
|
package/src/shield/policy.js
CHANGED
|
@@ -42,12 +42,46 @@ export async function loadPolicies(path) {
|
|
|
42
42
|
return data;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
// ReDoS protection: regexes are loaded from a user-provided JSON policy file,
|
|
46
|
+
// so a malicious or buggy pattern (e.g. `(a+)+$`) could pin the CPU on a long
|
|
47
|
+
// input. We mitigate two ways:
|
|
48
|
+
// 1) Cap the maximum input length passed to any regex test to MAX_REGEX_INPUT
|
|
49
|
+
// bytes. Above that we truncate before testing. Real agent values
|
|
50
|
+
// (URLs, commands, queries) are well under this in practice.
|
|
51
|
+
// 2) Reject obviously dangerous patterns at compile time (heuristic).
|
|
52
|
+
//
|
|
53
|
+
// A future v0.5 may add a proper safe-regex-2 dependency for thorough analysis.
|
|
54
|
+
const MAX_REGEX_INPUT = 8192;
|
|
55
|
+
|
|
56
|
+
const SUSPICIOUS_REGEX_PATTERNS = [
|
|
57
|
+
/(\([^)]*[+*][^)]*\))[+*]/, // (x+)+ or (x*)* — classic catastrophic backtracking
|
|
58
|
+
/(\.\*){3,}/, // multiple .* in a row
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
function validateRegexString(src, where) {
|
|
62
|
+
if (typeof src !== 'string') {
|
|
63
|
+
throw new Error(`policy ${where}: regex must be a string`);
|
|
64
|
+
}
|
|
65
|
+
if (src.length > 2000) {
|
|
66
|
+
throw new Error(`policy ${where}: regex too long (>2000 chars)`);
|
|
67
|
+
}
|
|
68
|
+
for (const sus of SUSPICIOUS_REGEX_PATTERNS) {
|
|
69
|
+
if (sus.test(src)) {
|
|
70
|
+
throw new Error(`policy ${where}: regex looks vulnerable to catastrophic backtracking ("${src.slice(0, 60)}…"). Refusing to load.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return new RegExp(src);
|
|
74
|
+
}
|
|
75
|
+
|
|
45
76
|
function compileMatchRegexes(match) {
|
|
46
|
-
for (const condition of Object.
|
|
77
|
+
for (const [field, condition] of Object.entries(match)) {
|
|
47
78
|
if (condition && typeof condition === 'object') {
|
|
48
|
-
if (condition.regex) condition._regex =
|
|
49
|
-
if (condition.not_regex) condition._not_regex =
|
|
50
|
-
if (condition.regex_any)
|
|
79
|
+
if (condition.regex) condition._regex = validateRegexString(condition.regex, `${field}.regex`);
|
|
80
|
+
if (condition.not_regex) condition._not_regex = validateRegexString(condition.not_regex, `${field}.not_regex`);
|
|
81
|
+
if (condition.regex_any) {
|
|
82
|
+
condition._regex_any = condition.regex_any.map((r, i) =>
|
|
83
|
+
validateRegexString(r, `${field}.regex_any[${i}]`));
|
|
84
|
+
}
|
|
51
85
|
}
|
|
52
86
|
}
|
|
53
87
|
}
|
|
@@ -56,6 +90,15 @@ function getNested(obj, path) {
|
|
|
56
90
|
return path.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
|
|
57
91
|
}
|
|
58
92
|
|
|
93
|
+
// Truncate input before passing to regex test — guards against ReDoS on
|
|
94
|
+
// pathologically long values (e.g. an agent that pastes a 5MB string into
|
|
95
|
+
// a tool argument).
|
|
96
|
+
function safeRegexTest(re, value) {
|
|
97
|
+
if (typeof value !== 'string') return false;
|
|
98
|
+
const s = value.length > MAX_REGEX_INPUT ? value.slice(0, MAX_REGEX_INPUT) : value;
|
|
99
|
+
return re.test(s);
|
|
100
|
+
}
|
|
101
|
+
|
|
59
102
|
function matchValue(value, condition) {
|
|
60
103
|
// Literal scalar match
|
|
61
104
|
if (condition === null || typeof condition !== 'object') {
|
|
@@ -67,13 +110,13 @@ function matchValue(value, condition) {
|
|
|
67
110
|
if (condition.in !== undefined) return condition.in.includes(value);
|
|
68
111
|
if (condition.not_in !== undefined) return !condition.not_in.includes(value);
|
|
69
112
|
if (condition._regex !== undefined) {
|
|
70
|
-
return
|
|
113
|
+
return safeRegexTest(condition._regex, value);
|
|
71
114
|
}
|
|
72
115
|
if (condition._not_regex !== undefined) {
|
|
73
|
-
return typeof value === 'string' && !condition._not_regex
|
|
116
|
+
return typeof value === 'string' && !safeRegexTest(condition._not_regex, value);
|
|
74
117
|
}
|
|
75
118
|
if (condition._regex_any !== undefined) {
|
|
76
|
-
return
|
|
119
|
+
return condition._regex_any.some(r => safeRegexTest(r, value));
|
|
77
120
|
}
|
|
78
121
|
// Unknown condition shape — defensive: fail-closed (no match) so unknown
|
|
79
122
|
// conditions never silently allow events.
|