watchmyagents 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -5
- package/SECURITY.md +9 -3
- package/package.json +13 -18
- package/scripts/anonymize.js +121 -0
- package/scripts/fetch-anthropic.js +10 -0
- package/scripts/inspect.js +14 -4
- package/scripts/shield.js +41 -4
- package/scripts/upload-fortress.js +211 -0
- package/src/anonymizer.js +198 -40
- package/src/logger.js +11 -2
- package/src/shield/policy.js +50 -7
- package/src/adapters/claude.js +0 -46
- package/src/adapters/generic.js +0 -21
- package/src/adapters/langchain.js +0 -42
- package/src/adapters/openai.js +0 -47
- package/src/collector.js +0 -113
- package/src/exporter.js +0 -71
- package/src/index.cjs +0 -36
- package/src/index.js +0 -26
package/src/anonymizer.js
CHANGED
|
@@ -1,48 +1,206 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
1
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
2
|
+
// Anonymizer — strip raw payloads, produce signals safe for Fortress
|
|
3
|
+
// ─────────────────────────────────────────────────────────────────────────
|
|
4
|
+
// Reads a Watch NDJSON file (the full local log) and produces an
|
|
5
|
+
// anonymized signals payload — the shape Fortress's `signals` table
|
|
6
|
+
// expects. The output contains ONLY:
|
|
7
|
+
//
|
|
8
|
+
// - counts (action_type, tool_name)
|
|
9
|
+
// - latencies (p50, p95, max) per tool
|
|
10
|
+
// - error rates per tool
|
|
11
|
+
// - salted SHA-256 hashes of IoCs (URLs, commands, queries)
|
|
12
|
+
// - top action_type sequences (Markov pairs)
|
|
13
|
+
// - stop_reason type counts (NOT the message text)
|
|
14
|
+
// - tokens_total
|
|
15
|
+
//
|
|
16
|
+
// What it NEVER outputs:
|
|
17
|
+
// - input.content (prompts)
|
|
18
|
+
// - output.content (agent text)
|
|
19
|
+
// - raw URLs / commands / queries
|
|
20
|
+
// - error messages
|
|
21
|
+
// - readable session_id (hashed)
|
|
22
|
+
// - readable agent_id (hashed)
|
|
23
|
+
// - PII of any kind
|
|
24
|
+
//
|
|
25
|
+
// This is the single bottleneck between Watch (local) and Fortress (cloud).
|
|
26
|
+
// Every byte that crosses to the cloud passes through this module.
|
|
27
|
+
|
|
28
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
29
|
+
import { createReadStream } from 'node:fs';
|
|
30
|
+
import { createInterface } from 'node:readline';
|
|
31
|
+
|
|
32
|
+
// ── Configuration ────────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
// Fields that may contain raw data — we extract a hash, never the raw value.
|
|
35
|
+
const HASHABLE_INPUT_FIELDS = ['url', 'query', 'command', 'path', 'file_path'];
|
|
36
|
+
|
|
37
|
+
// Tool types whose inputs we want to hash for IoC tracking
|
|
38
|
+
const TOOL_ACTIONS = new Set(['tool_use', 'mcp_tool_use', 'custom_tool_use']);
|
|
39
|
+
|
|
40
|
+
// ── Hash helpers ─────────────────────────────────────────────────────────
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Salted SHA-256 hash. The salt is per-customer (passed in) so the same URL
|
|
44
|
+
* at customer A produces a different hash than at customer B by default —
|
|
45
|
+
* but if a global salt is used, identical IoCs across customers produce
|
|
46
|
+
* identical hashes (the antivirus model for L4 cross-customer intel).
|
|
47
|
+
*/
|
|
48
|
+
export function hashWithSalt(value, salt) {
|
|
49
|
+
if (value == null) return null;
|
|
50
|
+
const s = typeof value === 'string' ? value : JSON.stringify(value);
|
|
51
|
+
return 'sha256:' + createHash('sha256').update(salt).update(s).digest('hex').slice(0, 32);
|
|
23
52
|
}
|
|
24
53
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return
|
|
54
|
+
// Generate a customer salt (if none provided)
|
|
55
|
+
export function generateSalt() {
|
|
56
|
+
return randomBytes(16).toString('hex');
|
|
28
57
|
}
|
|
29
58
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
if (
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
if (
|
|
38
|
-
out
|
|
39
|
-
} else if (typeof v === 'string') {
|
|
40
|
-
out[k] = scrubString(v);
|
|
41
|
-
} else if (typeof v === 'object') {
|
|
42
|
-
out[k] = anonymize(v);
|
|
43
|
-
} else {
|
|
44
|
-
out[k] = v;
|
|
59
|
+
// ── Single-entry extractor: what hashable IoCs are in this entry? ────────
|
|
60
|
+
|
|
61
|
+
function extractIocs(entry, salt) {
|
|
62
|
+
const out = [];
|
|
63
|
+
if (!entry.input || typeof entry.input !== 'object') return out;
|
|
64
|
+
for (const field of HASHABLE_INPUT_FIELDS) {
|
|
65
|
+
const v = entry.input[field];
|
|
66
|
+
if (typeof v === 'string' && v.length > 0) {
|
|
67
|
+
out.push(hashWithSalt(v, salt));
|
|
45
68
|
}
|
|
46
69
|
}
|
|
47
70
|
return out;
|
|
48
71
|
}
|
|
72
|
+
|
|
73
|
+
// ── Aggregator: walks the NDJSON stream and builds the signals payload ──
|
|
74
|
+
|
|
75
|
+
export class SignalsAggregator {
|
|
76
|
+
constructor({ salt } = {}) {
|
|
77
|
+
if (!salt) throw new Error('SignalsAggregator requires a salt');
|
|
78
|
+
this.salt = salt;
|
|
79
|
+
this.counts = Object.create(null); // action_type → count
|
|
80
|
+
this.toolCounts = Object.create(null); // tool_name → count
|
|
81
|
+
this.toolErrors = Object.create(null); // tool_name → error count
|
|
82
|
+
this.toolLatencies = Object.create(null); // tool_name → number[]
|
|
83
|
+
this.iocHashes = new Set(); // unique IoC hashes
|
|
84
|
+
this.sequences = Object.create(null); // "A → B" → count
|
|
85
|
+
this.stopReasons = Object.create(null); // stop_reason.type → count
|
|
86
|
+
this.tokensTotal = 0;
|
|
87
|
+
this.windowStart = null;
|
|
88
|
+
this.windowEnd = null;
|
|
89
|
+
this.entryCount = 0;
|
|
90
|
+
this._prevActionType = null;
|
|
91
|
+
this._prevSessionId = null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
add(entry) {
|
|
95
|
+
if (!entry) return;
|
|
96
|
+
this.entryCount++;
|
|
97
|
+
|
|
98
|
+
// Track window bounds
|
|
99
|
+
const ts = entry.timestamp || '';
|
|
100
|
+
if (ts) {
|
|
101
|
+
if (!this.windowStart || ts < this.windowStart) this.windowStart = ts;
|
|
102
|
+
if (!this.windowEnd || ts > this.windowEnd) this.windowEnd = ts;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Counts
|
|
106
|
+
const at = entry.action_type || 'unknown';
|
|
107
|
+
this.counts[at] = (this.counts[at] || 0) + 1;
|
|
108
|
+
|
|
109
|
+
// Sequence tracking (only within the same session)
|
|
110
|
+
if (this._prevActionType && entry.session_id === this._prevSessionId
|
|
111
|
+
&& at !== 'session_end' && this._prevActionType !== 'session_end') {
|
|
112
|
+
const seqKey = `${this._prevActionType} → ${at}`;
|
|
113
|
+
this.sequences[seqKey] = (this.sequences[seqKey] || 0) + 1;
|
|
114
|
+
}
|
|
115
|
+
this._prevActionType = at;
|
|
116
|
+
this._prevSessionId = entry.session_id || null;
|
|
117
|
+
|
|
118
|
+
// Tools
|
|
119
|
+
if (entry.tool_name && TOOL_ACTIONS.has(at)) {
|
|
120
|
+
this.toolCounts[entry.tool_name] = (this.toolCounts[entry.tool_name] || 0) + 1;
|
|
121
|
+
if (entry.status === 'error') {
|
|
122
|
+
this.toolErrors[entry.tool_name] = (this.toolErrors[entry.tool_name] || 0) + 1;
|
|
123
|
+
}
|
|
124
|
+
if (typeof entry.duration_ms === 'number') {
|
|
125
|
+
if (!this.toolLatencies[entry.tool_name]) this.toolLatencies[entry.tool_name] = [];
|
|
126
|
+
this.toolLatencies[entry.tool_name].push(entry.duration_ms);
|
|
127
|
+
}
|
|
128
|
+
// Extract & hash IoCs from this tool's input
|
|
129
|
+
for (const h of extractIocs(entry, this.salt)) this.iocHashes.add(h);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Tokens
|
|
133
|
+
if (typeof entry.tokens_used === 'number') this.tokensTotal += entry.tokens_used;
|
|
134
|
+
|
|
135
|
+
// Stop reasons (state_transition entries carry these)
|
|
136
|
+
const stopType = entry.output?.stop_reason?.type;
|
|
137
|
+
if (typeof stopType === 'string') {
|
|
138
|
+
this.stopReasons[stopType] = (this.stopReasons[stopType] || 0) + 1;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Compute p50/p95/max for an array of durations
|
|
143
|
+
_percentiles(arr) {
|
|
144
|
+
if (arr.length === 0) return null;
|
|
145
|
+
const sorted = [...arr].sort((a, b) => a - b);
|
|
146
|
+
const at = (p) => sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))];
|
|
147
|
+
return { p50: at(50), p95: at(95), max: sorted[sorted.length - 1], n: sorted.length };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
finalize() {
|
|
151
|
+
// Latencies aggregated
|
|
152
|
+
const latencies_p50_ms = {};
|
|
153
|
+
const latencies_p95_ms = {};
|
|
154
|
+
const error_rate_by_tool = {};
|
|
155
|
+
for (const [tool, durations] of Object.entries(this.toolLatencies)) {
|
|
156
|
+
const p = this._percentiles(durations);
|
|
157
|
+
if (p) {
|
|
158
|
+
latencies_p50_ms[tool] = p.p50;
|
|
159
|
+
latencies_p95_ms[tool] = p.p95;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
for (const tool of Object.keys(this.toolCounts)) {
|
|
163
|
+
const errs = this.toolErrors[tool] || 0;
|
|
164
|
+
error_rate_by_tool[tool] = +(errs / this.toolCounts[tool]).toFixed(4);
|
|
165
|
+
}
|
|
166
|
+
// Top-10 sequences
|
|
167
|
+
const sequencesTop = Object.entries(this.sequences)
|
|
168
|
+
.sort((a, b) => b[1] - a[1])
|
|
169
|
+
.slice(0, 10)
|
|
170
|
+
.map(([pattern, count]) => ({ pattern, count }));
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
window_start: this.windowStart,
|
|
174
|
+
window_end: this.windowEnd,
|
|
175
|
+
payload: {
|
|
176
|
+
counts: this.counts,
|
|
177
|
+
tool_counts: this.toolCounts,
|
|
178
|
+
latencies_p50_ms,
|
|
179
|
+
latencies_p95_ms,
|
|
180
|
+
error_rate_by_tool,
|
|
181
|
+
ioc_hashes: [...this.iocHashes],
|
|
182
|
+
sequences_top10: sequencesTop,
|
|
183
|
+
stop_reasons: this.stopReasons,
|
|
184
|
+
tokens_total: this.tokensTotal,
|
|
185
|
+
},
|
|
186
|
+
_meta: {
|
|
187
|
+
entries_processed: this.entryCount,
|
|
188
|
+
},
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// ── Streaming convenience: anonymize a whole NDJSON file/dir ────────────
|
|
194
|
+
|
|
195
|
+
export async function anonymizeFile(filePath, { salt } = {}) {
|
|
196
|
+
if (!salt) throw new Error('anonymizeFile requires a salt');
|
|
197
|
+
const agg = new SignalsAggregator({ salt });
|
|
198
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
199
|
+
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
200
|
+
for await (const line of rl) {
|
|
201
|
+
if (!line.trim()) continue;
|
|
202
|
+
let e; try { e = JSON.parse(line); } catch { continue; }
|
|
203
|
+
agg.add(e);
|
|
204
|
+
}
|
|
205
|
+
return agg.finalize();
|
|
206
|
+
}
|
package/src/logger.js
CHANGED
|
@@ -12,11 +12,17 @@ const EXPORT_FIELDS = [
|
|
|
12
12
|
];
|
|
13
13
|
|
|
14
14
|
export class Logger {
|
|
15
|
-
|
|
15
|
+
// `silent` : don't print log errors to stderr (default: true — quiet operation)
|
|
16
|
+
// `bestEffort` : SWALLOW write failures (default: false — fail loud).
|
|
17
|
+
// Audit-grade default: refuse to silently lose events. Disk
|
|
18
|
+
// full / EACCES / EINVAL must propagate so callers know.
|
|
19
|
+
// Opt into bestEffort=true only for non-critical paths.
|
|
20
|
+
constructor({ logDir, agentId, sessionId, silent, bestEffort } = {}) {
|
|
16
21
|
this.logDir = logDir;
|
|
17
22
|
this.agentId = agentId;
|
|
18
23
|
this.sessionId = sessionId || randomUUID();
|
|
19
24
|
this.silent = silent !== false;
|
|
25
|
+
this.bestEffort = bestEffort === true;
|
|
20
26
|
this.sequence = 0;
|
|
21
27
|
this.currentDay = null;
|
|
22
28
|
this.currentPath = null;
|
|
@@ -63,7 +69,10 @@ export class Logger {
|
|
|
63
69
|
await appendFile(path, JSON.stringify(full) + '\n', { encoding: 'utf8', mode: 0o600 });
|
|
64
70
|
this.count++;
|
|
65
71
|
} catch (err) {
|
|
66
|
-
if (!this.silent) process.stderr.write(`[wma] log error: ${err.message}\n`);
|
|
72
|
+
if (!this.silent) process.stderr.write(`[wma] log write error: ${err.message}\n`);
|
|
73
|
+
// Audit-grade default: fail loud so callers know events are being lost.
|
|
74
|
+
// Disk full, EACCES, EINVAL etc. should NOT be silently swallowed.
|
|
75
|
+
if (!this.bestEffort) throw err;
|
|
67
76
|
}
|
|
68
77
|
return full;
|
|
69
78
|
}
|
package/src/shield/policy.js
CHANGED
|
@@ -42,12 +42,46 @@ export async function loadPolicies(path) {
|
|
|
42
42
|
return data;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
// ReDoS protection: regexes are loaded from a user-provided JSON policy file,
|
|
46
|
+
// so a malicious or buggy pattern (e.g. `(a+)+$`) could pin the CPU on a long
|
|
47
|
+
// input. We mitigate two ways:
|
|
48
|
+
// 1) Cap the maximum input length passed to any regex test to MAX_REGEX_INPUT
|
|
49
|
+
// bytes. Above that we truncate before testing. Real agent values
|
|
50
|
+
// (URLs, commands, queries) are well under this in practice.
|
|
51
|
+
// 2) Reject obviously dangerous patterns at compile time (heuristic).
|
|
52
|
+
//
|
|
53
|
+
// A future v0.5 may add a proper safe-regex-2 dependency for thorough analysis.
|
|
54
|
+
const MAX_REGEX_INPUT = 8192;
|
|
55
|
+
|
|
56
|
+
const SUSPICIOUS_REGEX_PATTERNS = [
|
|
57
|
+
/(\([^)]*[+*][^)]*\))[+*]/, // (x+)+ or (x*)* — classic catastrophic backtracking
|
|
58
|
+
/(\.\*){3,}/, // multiple .* in a row
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
function validateRegexString(src, where) {
|
|
62
|
+
if (typeof src !== 'string') {
|
|
63
|
+
throw new Error(`policy ${where}: regex must be a string`);
|
|
64
|
+
}
|
|
65
|
+
if (src.length > 2000) {
|
|
66
|
+
throw new Error(`policy ${where}: regex too long (>2000 chars)`);
|
|
67
|
+
}
|
|
68
|
+
for (const sus of SUSPICIOUS_REGEX_PATTERNS) {
|
|
69
|
+
if (sus.test(src)) {
|
|
70
|
+
throw new Error(`policy ${where}: regex looks vulnerable to catastrophic backtracking ("${src.slice(0, 60)}…"). Refusing to load.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return new RegExp(src);
|
|
74
|
+
}
|
|
75
|
+
|
|
45
76
|
function compileMatchRegexes(match) {
|
|
46
|
-
for (const condition of Object.
|
|
77
|
+
for (const [field, condition] of Object.entries(match)) {
|
|
47
78
|
if (condition && typeof condition === 'object') {
|
|
48
|
-
if (condition.regex) condition._regex =
|
|
49
|
-
if (condition.not_regex) condition._not_regex =
|
|
50
|
-
if (condition.regex_any)
|
|
79
|
+
if (condition.regex) condition._regex = validateRegexString(condition.regex, `${field}.regex`);
|
|
80
|
+
if (condition.not_regex) condition._not_regex = validateRegexString(condition.not_regex, `${field}.not_regex`);
|
|
81
|
+
if (condition.regex_any) {
|
|
82
|
+
condition._regex_any = condition.regex_any.map((r, i) =>
|
|
83
|
+
validateRegexString(r, `${field}.regex_any[${i}]`));
|
|
84
|
+
}
|
|
51
85
|
}
|
|
52
86
|
}
|
|
53
87
|
}
|
|
@@ -56,6 +90,15 @@ function getNested(obj, path) {
|
|
|
56
90
|
return path.split('.').reduce((o, k) => (o == null ? undefined : o[k]), obj);
|
|
57
91
|
}
|
|
58
92
|
|
|
93
|
+
// Truncate input before passing to regex test — guards against ReDoS on
|
|
94
|
+
// pathologically long values (e.g. an agent that pastes a 5MB string into
|
|
95
|
+
// a tool argument).
|
|
96
|
+
function safeRegexTest(re, value) {
|
|
97
|
+
if (typeof value !== 'string') return false;
|
|
98
|
+
const s = value.length > MAX_REGEX_INPUT ? value.slice(0, MAX_REGEX_INPUT) : value;
|
|
99
|
+
return re.test(s);
|
|
100
|
+
}
|
|
101
|
+
|
|
59
102
|
function matchValue(value, condition) {
|
|
60
103
|
// Literal scalar match
|
|
61
104
|
if (condition === null || typeof condition !== 'object') {
|
|
@@ -67,13 +110,13 @@ function matchValue(value, condition) {
|
|
|
67
110
|
if (condition.in !== undefined) return condition.in.includes(value);
|
|
68
111
|
if (condition.not_in !== undefined) return !condition.not_in.includes(value);
|
|
69
112
|
if (condition._regex !== undefined) {
|
|
70
|
-
return
|
|
113
|
+
return safeRegexTest(condition._regex, value);
|
|
71
114
|
}
|
|
72
115
|
if (condition._not_regex !== undefined) {
|
|
73
|
-
return typeof value === 'string' && !condition._not_regex
|
|
116
|
+
return typeof value === 'string' && !safeRegexTest(condition._not_regex, value);
|
|
74
117
|
}
|
|
75
118
|
if (condition._regex_any !== undefined) {
|
|
76
|
-
return
|
|
119
|
+
return condition._regex_any.some(r => safeRegexTest(r, value));
|
|
77
120
|
}
|
|
78
121
|
// Unknown condition shape — defensive: fail-closed (no match) so unknown
|
|
79
122
|
// conditions never silently allow events.
|
package/src/adapters/claude.js
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import { WatchMyAgents } from '../collector.js';
|
|
2
|
-
|
|
3
|
-
export function createClaudeMonitor(opts = {}) {
|
|
4
|
-
const wma = WatchMyAgents.current() || new WatchMyAgents({ ...opts, framework: 'claude' });
|
|
5
|
-
|
|
6
|
-
return {
|
|
7
|
-
wma,
|
|
8
|
-
wrap(client) {
|
|
9
|
-
const m = client?.messages;
|
|
10
|
-
if (!m?.create) return client;
|
|
11
|
-
const orig = m.create.bind(m);
|
|
12
|
-
m.create = async (params) => {
|
|
13
|
-
const start = Date.now();
|
|
14
|
-
let status = 'ok', error = null, res;
|
|
15
|
-
try { res = await orig(params); return res; }
|
|
16
|
-
catch (e) { status = 'error'; error = e?.message || String(e); throw e; }
|
|
17
|
-
finally {
|
|
18
|
-
const u = res?.usage || {};
|
|
19
|
-
const inT = u.input_tokens || 0;
|
|
20
|
-
const outT = u.output_tokens || 0;
|
|
21
|
-
const cr = u.cache_read_input_tokens || 0;
|
|
22
|
-
const cw = u.cache_creation_input_tokens || 0;
|
|
23
|
-
const toolUses = Array.isArray(res?.content)
|
|
24
|
-
? res.content.filter(b => b?.type === 'tool_use').map(b => b.name) : [];
|
|
25
|
-
await wma.logAction({
|
|
26
|
-
framework: 'claude', action_type: 'llm_call',
|
|
27
|
-
tool_name: params?.model || 'messages.create',
|
|
28
|
-
model: params?.model || null,
|
|
29
|
-
duration_ms: Date.now() - start,
|
|
30
|
-
input_tokens: inT || null,
|
|
31
|
-
output_tokens: outT || null,
|
|
32
|
-
cache_read_tokens: cr || null,
|
|
33
|
-
cache_creation_tokens: cw || null,
|
|
34
|
-
tokens_used: (inT + outT + cr + cw) || null,
|
|
35
|
-
status, error,
|
|
36
|
-
input: { model: params?.model, message_count: params?.messages?.length, tool_count: params?.tools?.length || 0 },
|
|
37
|
-
output: { stop_reason: res?.stop_reason || null, tool_uses: toolUses },
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
};
|
|
41
|
-
return client;
|
|
42
|
-
},
|
|
43
|
-
logToolUse: (name, input, output, duration_ms) =>
|
|
44
|
-
wma.logAction({ framework: 'claude', action_type: 'tool_use', tool_name: name, duration_ms: duration_ms ?? null, status: 'ok', input, output }),
|
|
45
|
-
};
|
|
46
|
-
}
|
package/src/adapters/generic.js
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { WatchMyAgents } from '../collector.js';
|
|
2
|
-
|
|
3
|
-
export async function watch(toolName, params, fn, meta = {}) {
|
|
4
|
-
const wma = WatchMyAgents.getOrCreate();
|
|
5
|
-
return wma.watch(toolName, params, fn, { framework: 'generic', ...meta });
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
export function createGenericMonitor(opts = {}) {
|
|
9
|
-
const wma = WatchMyAgents.current() || new WatchMyAgents(opts);
|
|
10
|
-
return {
|
|
11
|
-
watch: (toolName, params, fn, meta) => wma.watch(toolName, params, fn, { framework: 'generic', ...meta }),
|
|
12
|
-
wrap(obj, methodNames) {
|
|
13
|
-
const names = methodNames || Object.keys(obj).filter(k => typeof obj[k] === 'function');
|
|
14
|
-
const wrapped = {};
|
|
15
|
-
for (const name of names) {
|
|
16
|
-
wrapped[name] = (...args) => wma.watch(name, args, () => obj[name](...args), { framework: 'generic' });
|
|
17
|
-
}
|
|
18
|
-
return { ...obj, ...wrapped };
|
|
19
|
-
},
|
|
20
|
-
};
|
|
21
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import { WatchMyAgents } from '../collector.js';
|
|
2
|
-
|
|
3
|
-
export function createLangChainHandler(opts = {}) {
|
|
4
|
-
const wma = WatchMyAgents.current() || new WatchMyAgents({ ...opts, framework: 'langchain' });
|
|
5
|
-
const starts = new Map();
|
|
6
|
-
const begin = id => starts.set(id, Date.now());
|
|
7
|
-
const elapsed = id => { const t = starts.get(id); starts.delete(id); return t ? Date.now() - t : null; };
|
|
8
|
-
|
|
9
|
-
return {
|
|
10
|
-
name: 'WatchMyAgentsHandler',
|
|
11
|
-
handleLLMStart: async (_l, _p, runId) => begin(runId),
|
|
12
|
-
handleLLMEnd: async (out, runId) => {
|
|
13
|
-
const u = out?.llmOutput?.tokenUsage || {};
|
|
14
|
-
const inT = u.promptTokens || 0, outT = u.completionTokens || 0;
|
|
15
|
-
return wma.logAction({
|
|
16
|
-
framework: 'langchain', action_type: 'llm_call', tool_name: 'llm',
|
|
17
|
-
model: out?.llmOutput?.modelName || null,
|
|
18
|
-
duration_ms: elapsed(runId),
|
|
19
|
-
input_tokens: inT || null, output_tokens: outT || null,
|
|
20
|
-
tokens_used: (inT + outT) || null, status: 'ok',
|
|
21
|
-
});
|
|
22
|
-
},
|
|
23
|
-
handleLLMError: async (err, runId) => wma.logAction({
|
|
24
|
-
framework: 'langchain', action_type: 'llm_call', tool_name: 'llm',
|
|
25
|
-
duration_ms: elapsed(runId), status: 'error', error: err?.message || String(err),
|
|
26
|
-
}),
|
|
27
|
-
handleToolStart: async (_t, _i, runId) => begin(runId),
|
|
28
|
-
handleToolEnd: async (_o, runId) => wma.logAction({
|
|
29
|
-
framework: 'langchain', action_type: 'tool_call', tool_name: 'tool',
|
|
30
|
-
duration_ms: elapsed(runId), status: 'ok',
|
|
31
|
-
}),
|
|
32
|
-
handleToolError: async (err, runId) => wma.logAction({
|
|
33
|
-
framework: 'langchain', action_type: 'tool_call', tool_name: 'tool',
|
|
34
|
-
duration_ms: elapsed(runId), status: 'error', error: err?.message || String(err),
|
|
35
|
-
}),
|
|
36
|
-
handleChainStart: async (_c, _i, runId) => begin(runId),
|
|
37
|
-
handleChainEnd: async (_o, runId) => wma.logAction({
|
|
38
|
-
framework: 'langchain', action_type: 'chain', tool_name: 'chain',
|
|
39
|
-
duration_ms: elapsed(runId), status: 'ok',
|
|
40
|
-
}),
|
|
41
|
-
};
|
|
42
|
-
}
|
package/src/adapters/openai.js
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { WatchMyAgents } from '../collector.js';
|
|
2
|
-
|
|
3
|
-
export function createOpenAIMonitor(opts = {}) {
|
|
4
|
-
const wma = WatchMyAgents.current() || new WatchMyAgents({ ...opts, framework: 'openai' });
|
|
5
|
-
|
|
6
|
-
function wrapMethod(obj, method, action_type) {
|
|
7
|
-
if (!obj || typeof obj[method] !== 'function') return;
|
|
8
|
-
const orig = obj[method].bind(obj);
|
|
9
|
-
obj[method] = async (params) => {
|
|
10
|
-
const start = Date.now();
|
|
11
|
-
let status = 'ok', error = null, res;
|
|
12
|
-
try { res = await orig(params); return res; }
|
|
13
|
-
catch (e) { status = 'error'; error = e?.message || String(e); throw e; }
|
|
14
|
-
finally {
|
|
15
|
-
const u = res?.usage || {};
|
|
16
|
-
const inT = u.prompt_tokens || u.input_tokens || 0;
|
|
17
|
-
const outT = u.completion_tokens || u.output_tokens || 0;
|
|
18
|
-
const cr = u.prompt_tokens_details?.cached_tokens || 0;
|
|
19
|
-
await wma.logAction({
|
|
20
|
-
framework: 'openai', action_type,
|
|
21
|
-
tool_name: params?.model || params?.assistant_id || method,
|
|
22
|
-
model: params?.model || null,
|
|
23
|
-
duration_ms: Date.now() - start,
|
|
24
|
-
input_tokens: inT || null,
|
|
25
|
-
output_tokens: outT || null,
|
|
26
|
-
cache_read_tokens: cr || null,
|
|
27
|
-
tokens_used: (inT + outT) || null,
|
|
28
|
-
status, error,
|
|
29
|
-
input: { model: params?.model, assistant_id: params?.assistant_id },
|
|
30
|
-
output: { id: res?.id, status: res?.status },
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
return {
|
|
37
|
-
wma,
|
|
38
|
-
wrap(client) {
|
|
39
|
-
wrapMethod(client?.chat?.completions, 'create', 'llm_call');
|
|
40
|
-
wrapMethod(client?.completions, 'create', 'llm_call');
|
|
41
|
-
wrapMethod(client?.beta?.threads?.runs, 'create', 'assistant_run');
|
|
42
|
-
wrapMethod(client?.beta?.threads?.runs, 'createAndPoll', 'assistant_run');
|
|
43
|
-
wrapMethod(client?.responses, 'create', 'llm_call');
|
|
44
|
-
return client;
|
|
45
|
-
},
|
|
46
|
-
};
|
|
47
|
-
}
|
package/src/collector.js
DELETED
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import { Logger } from './logger.js';
|
|
3
|
-
import { Exporter } from './exporter.js';
|
|
4
|
-
import { TokenTracker, estimateCost } from './tokens.js';
|
|
5
|
-
|
|
6
|
-
let _instance = null;
|
|
7
|
-
|
|
8
|
-
export class WatchMyAgents {
|
|
9
|
-
constructor(opts = {}) {
|
|
10
|
-
this.apiKey = opts.apiKey || process.env.WMA_API_KEY || null;
|
|
11
|
-
this.agentId = opts.agentId || 'default-agent';
|
|
12
|
-
this.logDir = opts.logDir || process.env.WMA_LOG_DIR || './watchmyagents-logs';
|
|
13
|
-
this.exportUrl = opts.exportUrl || process.env.WMA_EXPORT_URL || null;
|
|
14
|
-
this.silent = opts.silent !== false;
|
|
15
|
-
this.sessionId = opts.sessionId || randomUUID();
|
|
16
|
-
this.framework = opts.framework || 'generic';
|
|
17
|
-
this.tokenPricing = opts.tokenPricing || null;
|
|
18
|
-
this.tracker = new TokenTracker();
|
|
19
|
-
this.logger = new Logger({ logDir: this.logDir, agentId: this.agentId, sessionId: this.sessionId, silent: this.silent });
|
|
20
|
-
this.exporter = new Exporter({
|
|
21
|
-
apiKey: this.apiKey, exportUrl: this.exportUrl, agentId: this.agentId,
|
|
22
|
-
batchInterval: opts.batchInterval ?? 30000, silent: this.silent,
|
|
23
|
-
});
|
|
24
|
-
this.exporter.start();
|
|
25
|
-
_instance = this;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
static current() { return _instance; }
|
|
29
|
-
static getOrCreate(opts) { return _instance || new WatchMyAgents(opts); }
|
|
30
|
-
|
|
31
|
-
summarize(v) {
|
|
32
|
-
if (v == null) return null;
|
|
33
|
-
const t = typeof v;
|
|
34
|
-
if (t === 'string') return { type: 'string', length: v.length };
|
|
35
|
-
if (t === 'number' || t === 'boolean') return { type: t };
|
|
36
|
-
if (Array.isArray(v)) return { type: 'array', length: v.length };
|
|
37
|
-
if (t === 'object') return { type: 'object', keys: Object.keys(v).length };
|
|
38
|
-
return { type: t };
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
_enrichTokens(entry) {
|
|
42
|
-
const i = entry.input_tokens, o = entry.output_tokens;
|
|
43
|
-
const cr = entry.cache_read_tokens || 0, cw = entry.cache_creation_tokens || 0;
|
|
44
|
-
if (entry.tokens_used == null && (i != null || o != null)) {
|
|
45
|
-
entry.tokens_used = (i || 0) + (o || 0) + cr + cw;
|
|
46
|
-
}
|
|
47
|
-
if (entry.cost_usd == null && entry.model) {
|
|
48
|
-
entry.cost_usd = estimateCost(entry.model, {
|
|
49
|
-
input_tokens: i, output_tokens: o,
|
|
50
|
-
cache_read_tokens: cr, cache_creation_tokens: cw,
|
|
51
|
-
}, this.tokenPricing);
|
|
52
|
-
}
|
|
53
|
-
return entry;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async watch(toolName, params, fn, meta = {}) {
|
|
57
|
-
const start = Date.now();
|
|
58
|
-
const id = randomUUID();
|
|
59
|
-
let status = 'ok', error = null, result;
|
|
60
|
-
try { result = await fn(); return result; }
|
|
61
|
-
catch (e) { status = 'error'; error = e?.message || String(e); throw e; }
|
|
62
|
-
finally {
|
|
63
|
-
const entry = await this.logger.write(this._enrichTokens({
|
|
64
|
-
id, framework: meta.framework || this.framework,
|
|
65
|
-
action_type: meta.action_type || 'tool_call',
|
|
66
|
-
tool_name: toolName, duration_ms: Date.now() - start,
|
|
67
|
-
model: meta.model ?? null,
|
|
68
|
-
tokens_used: meta.tokens_used ?? null,
|
|
69
|
-
input_tokens: meta.input_tokens ?? null,
|
|
70
|
-
output_tokens: meta.output_tokens ?? null,
|
|
71
|
-
cache_read_tokens: meta.cache_read_tokens ?? null,
|
|
72
|
-
cache_creation_tokens: meta.cache_creation_tokens ?? null,
|
|
73
|
-
status, error, input: params, output: this.summarize(result),
|
|
74
|
-
}));
|
|
75
|
-
this.tracker.record(entry);
|
|
76
|
-
this.exporter.enqueue(this.logger.toExportRecord(entry));
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
async logAction(entry) {
|
|
81
|
-
const written = await this.logger.write(this._enrichTokens({ ...entry }));
|
|
82
|
-
this.tracker.record(written);
|
|
83
|
-
this.exporter.enqueue(this.logger.toExportRecord(written));
|
|
84
|
-
return written;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
tokenStats() { return this.tracker.stats(); }
|
|
88
|
-
|
|
89
|
-
async flush() { await this.exporter.flush(); }
|
|
90
|
-
|
|
91
|
-
async shutdown() {
|
|
92
|
-
const stats = this.tracker.stats().total;
|
|
93
|
-
const entry = await this.logger.write({
|
|
94
|
-
action_type: 'session_end',
|
|
95
|
-
tool_name: null,
|
|
96
|
-
framework: this.framework,
|
|
97
|
-
status: 'ok',
|
|
98
|
-
session_tokens: {
|
|
99
|
-
input: stats.input,
|
|
100
|
-
output: stats.output,
|
|
101
|
-
cache_read: stats.cache_read,
|
|
102
|
-
cache_creation: stats.cache_creation,
|
|
103
|
-
total: stats.sum,
|
|
104
|
-
},
|
|
105
|
-
session_cost_usd: stats.cost_usd,
|
|
106
|
-
});
|
|
107
|
-
this.exporter.enqueue(this.logger.toExportRecord(entry));
|
|
108
|
-
this.exporter.stop();
|
|
109
|
-
await this.exporter.flush();
|
|
110
|
-
}
|
|
111
|
-
get logPath() { return this.logger._pathForToday(); }
|
|
112
|
-
get actionCount() { return this.logger.count; }
|
|
113
|
-
}
|