trickle-observe 0.2.117 → 0.2.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fetch-observer.js +13 -2
- package/dist/llm-observer.d.ts +17 -0
- package/dist/llm-observer.js +526 -0
- package/dist/observe-register.js +21 -0
- package/dist/vite-plugin.test.d.ts +1 -0
- package/dist/vite-plugin.test.js +160 -0
- package/package.json +1 -1
- package/src/fetch-observer.ts +14 -2
- package/src/llm-observer.ts +508 -0
- package/src/observe-register.ts +22 -0
package/dist/fetch-observer.js
CHANGED
|
@@ -115,6 +115,18 @@ function patchFetch(environment, debugMode) {
|
|
|
115
115
|
// Mark as patched
|
|
116
116
|
globalThis.fetch.__trickle_patched = true;
|
|
117
117
|
}
|
|
118
|
+
/**
|
|
119
|
+
* Replace literal IDs in URL paths with placeholders to avoid cardinality explosion.
|
|
120
|
+
* "/users/abc123/tasks/456" → "/users/:id/tasks/:id"
|
|
121
|
+
* "/items/550e8400-e29b-41d4-a716-446655440000" → "/items/:uuid"
|
|
122
|
+
*/
|
|
123
|
+
function normalizePath(pathname) {
|
|
124
|
+
return pathname
|
|
125
|
+
.replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/:uuid')
|
|
126
|
+
.replace(/\/[0-9a-f]{24}(?=\/|$)/gi, '/:id')
|
|
127
|
+
.replace(/\/[0-9a-f]{8,}(?=\/|$)/gi, '/:id')
|
|
128
|
+
.replace(/\/\d+(?=\/|$)/g, '/:id');
|
|
129
|
+
}
|
|
118
130
|
/**
|
|
119
131
|
* Parse a URL into a clean function name and module name.
|
|
120
132
|
* "https://api.example.com/v1/users?limit=10"
|
|
@@ -123,14 +135,13 @@ function patchFetch(environment, debugMode) {
|
|
|
123
135
|
function parseUrl(method, rawUrl) {
|
|
124
136
|
try {
|
|
125
137
|
const parsed = new URL(rawUrl);
|
|
126
|
-
const pathname = parsed.pathname || '/';
|
|
138
|
+
const pathname = normalizePath(parsed.pathname || '/');
|
|
127
139
|
return {
|
|
128
140
|
functionName: `${method} ${pathname}`,
|
|
129
141
|
module: parsed.hostname || 'http',
|
|
130
142
|
};
|
|
131
143
|
}
|
|
132
144
|
catch {
|
|
133
|
-
// Relative URL or invalid — use as-is
|
|
134
145
|
return {
|
|
135
146
|
functionName: `${method} ${rawUrl}`,
|
|
136
147
|
module: 'http',
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM call observer — auto-instruments OpenAI, Anthropic, and other LLM SDKs
|
|
3
|
+
* to capture prompts, completions, token counts, latency, cost, and model metadata.
|
|
4
|
+
*
|
|
5
|
+
* Writes to .trickle/llm.jsonl as:
|
|
6
|
+
* { "kind": "llm_call", "provider": "openai", "model": "gpt-4",
|
|
7
|
+
* "inputTokens": 100, "outputTokens": 50, "durationMs": 1234.5, ... }
|
|
8
|
+
*
|
|
9
|
+
* Supports both streaming and non-streaming calls.
|
|
10
|
+
* Zero code changes needed — intercepted via Module._load hook.
|
|
11
|
+
*/
|
|
12
|
+
export declare function patchOpenAI(openaiModule: any, debug: boolean): void;
|
|
13
|
+
export declare function patchAnthropic(anthropicModule: any, debug: boolean): void;
|
|
14
|
+
/**
|
|
15
|
+
* Initialize the LLM observer — clears previous data file.
|
|
16
|
+
*/
|
|
17
|
+
export declare function initLlmObserver(): void;
|
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* LLM call observer — auto-instruments OpenAI, Anthropic, and other LLM SDKs
|
|
4
|
+
* to capture prompts, completions, token counts, latency, cost, and model metadata.
|
|
5
|
+
*
|
|
6
|
+
* Writes to .trickle/llm.jsonl as:
|
|
7
|
+
* { "kind": "llm_call", "provider": "openai", "model": "gpt-4",
|
|
8
|
+
* "inputTokens": 100, "outputTokens": 50, "durationMs": 1234.5, ... }
|
|
9
|
+
*
|
|
10
|
+
* Supports both streaming and non-streaming calls.
|
|
11
|
+
* Zero code changes needed — intercepted via Module._load hook.
|
|
12
|
+
*/
|
|
13
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
16
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
17
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
18
|
+
}
|
|
19
|
+
Object.defineProperty(o, k2, desc);
|
|
20
|
+
}) : (function(o, m, k, k2) {
|
|
21
|
+
if (k2 === undefined) k2 = k;
|
|
22
|
+
o[k2] = m[k];
|
|
23
|
+
}));
|
|
24
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
25
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
26
|
+
}) : function(o, v) {
|
|
27
|
+
o["default"] = v;
|
|
28
|
+
});
|
|
29
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
30
|
+
var ownKeys = function(o) {
|
|
31
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
32
|
+
var ar = [];
|
|
33
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
34
|
+
return ar;
|
|
35
|
+
};
|
|
36
|
+
return ownKeys(o);
|
|
37
|
+
};
|
|
38
|
+
return function (mod) {
|
|
39
|
+
if (mod && mod.__esModule) return mod;
|
|
40
|
+
var result = {};
|
|
41
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
42
|
+
__setModuleDefault(result, mod);
|
|
43
|
+
return result;
|
|
44
|
+
};
|
|
45
|
+
})();
|
|
46
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
47
|
+
exports.patchOpenAI = patchOpenAI;
|
|
48
|
+
exports.patchAnthropic = patchAnthropic;
|
|
49
|
+
exports.initLlmObserver = initLlmObserver;
|
|
50
|
+
const fs = __importStar(require("fs"));
|
|
51
|
+
const path = __importStar(require("path"));
|
|
52
|
+
let llmFile = null;
|
|
53
|
+
let eventCount = 0;
|
|
54
|
+
const MAX_LLM_EVENTS = 500;
|
|
55
|
+
const TRUNCATE_LEN = 500;
|
|
56
|
+
// Approximate pricing per 1M tokens (USD) — used for cost estimation
|
|
57
|
+
const PRICING = {
|
|
58
|
+
'gpt-4o': { input: 2.5, output: 10 },
|
|
59
|
+
'gpt-4o-mini': { input: 0.15, output: 0.6 },
|
|
60
|
+
'gpt-4-turbo': { input: 10, output: 30 },
|
|
61
|
+
'gpt-4': { input: 30, output: 60 },
|
|
62
|
+
'gpt-3.5-turbo': { input: 0.5, output: 1.5 },
|
|
63
|
+
'claude-opus-4-20250514': { input: 15, output: 75 },
|
|
64
|
+
'claude-sonnet-4-20250514': { input: 3, output: 15 },
|
|
65
|
+
'claude-3-5-sonnet-20241022': { input: 3, output: 15 },
|
|
66
|
+
'claude-3-5-haiku-20241022': { input: 0.8, output: 4 },
|
|
67
|
+
'claude-3-haiku-20240307': { input: 0.25, output: 1.25 },
|
|
68
|
+
};
|
|
69
|
+
function getLlmFile() {
|
|
70
|
+
if (llmFile)
|
|
71
|
+
return llmFile;
|
|
72
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
73
|
+
try {
|
|
74
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
75
|
+
}
|
|
76
|
+
catch { }
|
|
77
|
+
llmFile = path.join(dir, 'llm.jsonl');
|
|
78
|
+
return llmFile;
|
|
79
|
+
}
|
|
80
|
+
function writeLlmEvent(event) {
|
|
81
|
+
if (eventCount >= MAX_LLM_EVENTS)
|
|
82
|
+
return;
|
|
83
|
+
eventCount++;
|
|
84
|
+
try {
|
|
85
|
+
fs.appendFileSync(getLlmFile(), JSON.stringify(event) + '\n');
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
function truncate(s, len = TRUNCATE_LEN) {
|
|
90
|
+
if (!s)
|
|
91
|
+
return '';
|
|
92
|
+
return s.length > len ? s.substring(0, len) + '...' : s;
|
|
93
|
+
}
|
|
94
|
+
function estimateCost(model, inputTokens, outputTokens) {
|
|
95
|
+
// Find best matching pricing key
|
|
96
|
+
const key = Object.keys(PRICING).find(k => model.includes(k)) || '';
|
|
97
|
+
if (!key)
|
|
98
|
+
return 0;
|
|
99
|
+
const p = PRICING[key];
|
|
100
|
+
return Math.round(((inputTokens * p.input + outputTokens * p.output) / 1_000_000) * 1_000_000) / 1_000_000;
|
|
101
|
+
}
|
|
102
|
+
function extractInputPreview(messages) {
|
|
103
|
+
if (!Array.isArray(messages) || messages.length === 0)
|
|
104
|
+
return '';
|
|
105
|
+
const last = messages[messages.length - 1];
|
|
106
|
+
if (typeof last?.content === 'string')
|
|
107
|
+
return truncate(last.content);
|
|
108
|
+
if (Array.isArray(last?.content)) {
|
|
109
|
+
const textPart = last.content.find((p) => p.type === 'text');
|
|
110
|
+
if (textPart?.text)
|
|
111
|
+
return truncate(textPart.text);
|
|
112
|
+
}
|
|
113
|
+
return '';
|
|
114
|
+
}
|
|
115
|
+
function extractSystemPrompt(messages) {
|
|
116
|
+
if (!Array.isArray(messages))
|
|
117
|
+
return undefined;
|
|
118
|
+
const sys = messages.find((m) => m.role === 'system');
|
|
119
|
+
if (sys?.content && typeof sys.content === 'string')
|
|
120
|
+
return truncate(sys.content, 200);
|
|
121
|
+
return undefined;
|
|
122
|
+
}
|
|
123
|
+
function hasToolUse(params) {
|
|
124
|
+
return !!(params.tools && Array.isArray(params.tools) && params.tools.length > 0);
|
|
125
|
+
}
|
|
126
|
+
// ────────────────────────────────────────────────────
|
|
127
|
+
// OpenAI SDK v4+ instrumentation
|
|
128
|
+
// ────────────────────────────────────────────────────
|
|
129
|
+
function patchOpenAI(openaiModule, debug) {
|
|
130
|
+
if (!openaiModule || getattr(openaiModule, '_trickle_llm_patched'))
|
|
131
|
+
return;
|
|
132
|
+
setattr(openaiModule, '_trickle_llm_patched', true);
|
|
133
|
+
const OpenAIClass = openaiModule.OpenAI || openaiModule.default;
|
|
134
|
+
if (typeof OpenAIClass !== 'function')
|
|
135
|
+
return;
|
|
136
|
+
// OpenAI SDK v4+ creates resource instances (chat, completions) in the constructor
|
|
137
|
+
// as own properties. The Completions class is not directly exported, but we can
|
|
138
|
+
// access it by creating a temporary client and getting the prototype of chat.completions.
|
|
139
|
+
try {
|
|
140
|
+
// Create a temporary client to discover the Completions class
|
|
141
|
+
// (ES6 classes require `new`, can't use .call())
|
|
142
|
+
const tmpClient = new OpenAIClass({ apiKey: 'trickle-probe' });
|
|
143
|
+
const CompletionsClass = Object.getPrototypeOf(tmpClient.chat?.completions)?.constructor;
|
|
144
|
+
if (CompletionsClass && CompletionsClass.prototype.create && !CompletionsClass.prototype.create.__trickle_patched) {
|
|
145
|
+
const origCreate = CompletionsClass.prototype.create;
|
|
146
|
+
CompletionsClass.prototype.create = function patchedCreate(...args) {
|
|
147
|
+
const params = args[0] || {};
|
|
148
|
+
const startTime = performance.now();
|
|
149
|
+
const isStream = !!params.stream;
|
|
150
|
+
const result = origCreate.apply(this, args);
|
|
151
|
+
if (isStream)
|
|
152
|
+
return handleOpenAIStream(result, params, startTime, debug);
|
|
153
|
+
if (result && typeof result.then === 'function') {
|
|
154
|
+
return result.then((response) => {
|
|
155
|
+
captureOpenAIResponse(params, response, startTime, debug);
|
|
156
|
+
return response;
|
|
157
|
+
}).catch((err) => {
|
|
158
|
+
captureOpenAIError(params, err, startTime, debug);
|
|
159
|
+
throw err;
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return result;
|
|
163
|
+
};
|
|
164
|
+
CompletionsClass.prototype.create.__trickle_patched = true;
|
|
165
|
+
if (debug)
|
|
166
|
+
console.log('[trickle/llm] Patched OpenAI SDK');
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch (e) {
|
|
171
|
+
if (debug)
|
|
172
|
+
console.log('[trickle/llm] OpenAI patch probe failed:', e.message);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function patchOpenAIClient(client, debug) {
|
|
176
|
+
// Patch chat.completions.create
|
|
177
|
+
if (client.chat?.completions?.create && !client.chat.completions.create.__trickle_patched) {
|
|
178
|
+
const origCreate = client.chat.completions.create.bind(client.chat.completions);
|
|
179
|
+
client.chat.completions.create = function patchedCreate(...args) {
|
|
180
|
+
const params = args[0] || {};
|
|
181
|
+
const startTime = performance.now();
|
|
182
|
+
const isStream = !!params.stream;
|
|
183
|
+
const result = origCreate(...args);
|
|
184
|
+
if (isStream) {
|
|
185
|
+
return handleOpenAIStream(result, params, startTime, debug);
|
|
186
|
+
}
|
|
187
|
+
// Non-streaming: hook the promise
|
|
188
|
+
if (result && typeof result.then === 'function') {
|
|
189
|
+
return result.then((response) => {
|
|
190
|
+
captureOpenAIResponse(params, response, startTime, debug);
|
|
191
|
+
return response;
|
|
192
|
+
}).catch((err) => {
|
|
193
|
+
captureOpenAIError(params, err, startTime, debug);
|
|
194
|
+
throw err;
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
return result;
|
|
198
|
+
};
|
|
199
|
+
client.chat.completions.create.__trickle_patched = true;
|
|
200
|
+
}
|
|
201
|
+
// Patch completions.create (legacy)
|
|
202
|
+
if (client.completions?.create && !client.completions.create.__trickle_patched) {
|
|
203
|
+
const origCreate = client.completions.create.bind(client.completions);
|
|
204
|
+
client.completions.create = function patchedCreate(...args) {
|
|
205
|
+
const params = args[0] || {};
|
|
206
|
+
const startTime = performance.now();
|
|
207
|
+
const result = origCreate(...args);
|
|
208
|
+
if (result && typeof result.then === 'function') {
|
|
209
|
+
return result.then((response) => {
|
|
210
|
+
const usage = response.usage || {};
|
|
211
|
+
const text = response.choices?.[0]?.text || '';
|
|
212
|
+
writeLlmEvent({
|
|
213
|
+
kind: 'llm_call', provider: 'openai', model: params.model || 'unknown',
|
|
214
|
+
durationMs: round(performance.now() - startTime),
|
|
215
|
+
inputTokens: usage.prompt_tokens || 0, outputTokens: usage.completion_tokens || 0,
|
|
216
|
+
totalTokens: usage.total_tokens || 0,
|
|
217
|
+
estimatedCostUsd: estimateCost(params.model || '', usage.prompt_tokens || 0, usage.completion_tokens || 0),
|
|
218
|
+
stream: false, finishReason: response.choices?.[0]?.finish_reason || 'unknown',
|
|
219
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
220
|
+
inputPreview: truncate(params.prompt || ''), outputPreview: truncate(text),
|
|
221
|
+
messageCount: 0, toolUse: false, timestamp: Date.now(),
|
|
222
|
+
});
|
|
223
|
+
return response;
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
return result;
|
|
227
|
+
};
|
|
228
|
+
client.completions.create.__trickle_patched = true;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
async function handleOpenAIStream(resultPromise, params, startTime, debug) {
|
|
232
|
+
const stream = await resultPromise;
|
|
233
|
+
const chunks = [];
|
|
234
|
+
let finishReason = 'unknown';
|
|
235
|
+
let totalInputTokens = 0;
|
|
236
|
+
let totalOutputTokens = 0;
|
|
237
|
+
// Wrap the async iterator
|
|
238
|
+
const origIterator = stream[Symbol.asyncIterator].bind(stream);
|
|
239
|
+
stream[Symbol.asyncIterator] = function () {
|
|
240
|
+
const iter = origIterator();
|
|
241
|
+
return {
|
|
242
|
+
async next() {
|
|
243
|
+
const result = await iter.next();
|
|
244
|
+
if (!result.done) {
|
|
245
|
+
const chunk = result.value;
|
|
246
|
+
const delta = chunk.choices?.[0]?.delta;
|
|
247
|
+
if (delta?.content)
|
|
248
|
+
chunks.push(delta.content);
|
|
249
|
+
if (chunk.choices?.[0]?.finish_reason)
|
|
250
|
+
finishReason = chunk.choices[0].finish_reason;
|
|
251
|
+
if (chunk.usage) {
|
|
252
|
+
totalInputTokens = chunk.usage.prompt_tokens || totalInputTokens;
|
|
253
|
+
totalOutputTokens = chunk.usage.completion_tokens || totalOutputTokens;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
else {
|
|
257
|
+
// Stream finished — capture
|
|
258
|
+
const outputText = chunks.join('');
|
|
259
|
+
writeLlmEvent({
|
|
260
|
+
kind: 'llm_call', provider: 'openai', model: params.model || 'unknown',
|
|
261
|
+
durationMs: round(performance.now() - startTime),
|
|
262
|
+
inputTokens: totalInputTokens, outputTokens: totalOutputTokens,
|
|
263
|
+
totalTokens: totalInputTokens + totalOutputTokens,
|
|
264
|
+
estimatedCostUsd: estimateCost(params.model || '', totalInputTokens, totalOutputTokens),
|
|
265
|
+
stream: true, finishReason,
|
|
266
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
267
|
+
systemPrompt: extractSystemPrompt(params.messages),
|
|
268
|
+
inputPreview: extractInputPreview(params.messages),
|
|
269
|
+
outputPreview: truncate(outputText),
|
|
270
|
+
messageCount: params.messages?.length || 0,
|
|
271
|
+
toolUse: hasToolUse(params), timestamp: Date.now(),
|
|
272
|
+
});
|
|
273
|
+
if (debug)
|
|
274
|
+
console.log(`[trickle/llm] OpenAI stream: ${params.model} (${totalOutputTokens} tokens)`);
|
|
275
|
+
}
|
|
276
|
+
return result;
|
|
277
|
+
},
|
|
278
|
+
return: iter.return?.bind(iter),
|
|
279
|
+
throw: iter.throw?.bind(iter),
|
|
280
|
+
};
|
|
281
|
+
};
|
|
282
|
+
return stream;
|
|
283
|
+
}
|
|
284
|
+
function captureOpenAIResponse(params, response, startTime, debug) {
|
|
285
|
+
const usage = response.usage || {};
|
|
286
|
+
const outputText = response.choices?.[0]?.message?.content || '';
|
|
287
|
+
const event = {
|
|
288
|
+
kind: 'llm_call', provider: 'openai', model: params.model || 'unknown',
|
|
289
|
+
durationMs: round(performance.now() - startTime),
|
|
290
|
+
inputTokens: usage.prompt_tokens || 0, outputTokens: usage.completion_tokens || 0,
|
|
291
|
+
totalTokens: usage.total_tokens || 0,
|
|
292
|
+
estimatedCostUsd: estimateCost(params.model || '', usage.prompt_tokens || 0, usage.completion_tokens || 0),
|
|
293
|
+
stream: false, finishReason: response.choices?.[0]?.finish_reason || 'unknown',
|
|
294
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
295
|
+
systemPrompt: extractSystemPrompt(params.messages),
|
|
296
|
+
inputPreview: extractInputPreview(params.messages),
|
|
297
|
+
outputPreview: truncate(outputText),
|
|
298
|
+
messageCount: params.messages?.length || 0,
|
|
299
|
+
toolUse: hasToolUse(params), timestamp: Date.now(),
|
|
300
|
+
};
|
|
301
|
+
writeLlmEvent(event);
|
|
302
|
+
if (debug)
|
|
303
|
+
console.log(`[trickle/llm] OpenAI: ${params.model} (${usage.total_tokens || 0} tokens, ${event.durationMs}ms)`);
|
|
304
|
+
}
|
|
305
|
+
function captureOpenAIError(params, err, startTime, debug) {
|
|
306
|
+
writeLlmEvent({
|
|
307
|
+
kind: 'llm_call', provider: 'openai', model: params.model || 'unknown',
|
|
308
|
+
durationMs: round(performance.now() - startTime),
|
|
309
|
+
inputTokens: 0, outputTokens: 0, totalTokens: 0, estimatedCostUsd: 0,
|
|
310
|
+
stream: !!params.stream, finishReason: 'error',
|
|
311
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
312
|
+
systemPrompt: extractSystemPrompt(params.messages),
|
|
313
|
+
inputPreview: extractInputPreview(params.messages),
|
|
314
|
+
outputPreview: '', messageCount: params.messages?.length || 0,
|
|
315
|
+
toolUse: hasToolUse(params), timestamp: Date.now(),
|
|
316
|
+
error: truncate(err?.message || String(err), 200),
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
// ────────────────────────────────────────────────────
|
|
320
|
+
// Anthropic SDK instrumentation
|
|
321
|
+
// ────────────────────────────────────────────────────
|
|
322
|
+
function patchAnthropic(anthropicModule, debug) {
|
|
323
|
+
if (!anthropicModule || getattr(anthropicModule, '_trickle_llm_patched'))
|
|
324
|
+
return;
|
|
325
|
+
setattr(anthropicModule, '_trickle_llm_patched', true);
|
|
326
|
+
const AnthropicClass = anthropicModule.Anthropic || anthropicModule.default;
|
|
327
|
+
if (typeof AnthropicClass !== 'function')
|
|
328
|
+
return;
|
|
329
|
+
try {
|
|
330
|
+
const tmpClient = new AnthropicClass({ apiKey: 'trickle-probe' });
|
|
331
|
+
const MessagesClass = Object.getPrototypeOf(tmpClient.messages)?.constructor;
|
|
332
|
+
if (MessagesClass && MessagesClass.prototype.create && !MessagesClass.prototype.create.__trickle_patched) {
|
|
333
|
+
const origCreate = MessagesClass.prototype.create;
|
|
334
|
+
MessagesClass.prototype.create = function patchedCreate(...args) {
|
|
335
|
+
const params = args[0] || {};
|
|
336
|
+
const startTime = performance.now();
|
|
337
|
+
const isStream = !!params.stream;
|
|
338
|
+
const result = origCreate.apply(this, args);
|
|
339
|
+
if (result && typeof result.then === 'function') {
|
|
340
|
+
return result.then((response) => {
|
|
341
|
+
if (isStream)
|
|
342
|
+
return handleAnthropicStream(response, params, startTime, debug);
|
|
343
|
+
captureAnthropicResponse(params, response, startTime, debug);
|
|
344
|
+
return response;
|
|
345
|
+
}).catch((err) => {
|
|
346
|
+
captureAnthropicError(params, err, startTime, debug);
|
|
347
|
+
throw err;
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
return result;
|
|
351
|
+
};
|
|
352
|
+
MessagesClass.prototype.create.__trickle_patched = true;
|
|
353
|
+
if (debug)
|
|
354
|
+
console.log('[trickle/llm] Patched Anthropic SDK');
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
catch (e) {
|
|
359
|
+
if (debug)
|
|
360
|
+
console.log('[trickle/llm] Anthropic patch probe failed:', e.message);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
function patchAnthropicClient(client, debug) {
|
|
364
|
+
// Patch messages.create
|
|
365
|
+
if (client.messages?.create && !client.messages.create.__trickle_patched) {
|
|
366
|
+
const origCreate = client.messages.create.bind(client.messages);
|
|
367
|
+
client.messages.create = function patchedCreate(...args) {
|
|
368
|
+
const params = args[0] || {};
|
|
369
|
+
const startTime = performance.now();
|
|
370
|
+
const isStream = !!params.stream;
|
|
371
|
+
const result = origCreate(...args);
|
|
372
|
+
if (result && typeof result.then === 'function') {
|
|
373
|
+
return result.then((response) => {
|
|
374
|
+
if (isStream) {
|
|
375
|
+
return handleAnthropicStream(response, params, startTime, debug);
|
|
376
|
+
}
|
|
377
|
+
captureAnthropicResponse(params, response, startTime, debug);
|
|
378
|
+
return response;
|
|
379
|
+
}).catch((err) => {
|
|
380
|
+
captureAnthropicError(params, err, startTime, debug);
|
|
381
|
+
throw err;
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
return result;
|
|
385
|
+
};
|
|
386
|
+
client.messages.create.__trickle_patched = true;
|
|
387
|
+
}
|
|
388
|
+
// Patch messages.stream (if it exists)
|
|
389
|
+
if (client.messages?.stream && !client.messages.stream.__trickle_patched) {
|
|
390
|
+
const origStream = client.messages.stream.bind(client.messages);
|
|
391
|
+
client.messages.stream = function patchedStream(...args) {
|
|
392
|
+
const params = args[0] || {};
|
|
393
|
+
const startTime = performance.now();
|
|
394
|
+
const result = origStream(...args);
|
|
395
|
+
if (result && typeof result.then === 'function') {
|
|
396
|
+
return result.then((stream) => handleAnthropicStream(stream, params, startTime, debug));
|
|
397
|
+
}
|
|
398
|
+
return handleAnthropicStream(result, params, startTime, debug);
|
|
399
|
+
};
|
|
400
|
+
client.messages.stream.__trickle_patched = true;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
function handleAnthropicStream(stream, params, startTime, debug) {
|
|
404
|
+
// Anthropic streams have a finalMessage() or on('message') pattern
|
|
405
|
+
// Hook into the stream events to capture the final result
|
|
406
|
+
if (stream && typeof stream.on === 'function') {
|
|
407
|
+
stream.on('finalMessage', (message) => {
|
|
408
|
+
captureAnthropicResponse(params, message, startTime, debug);
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
// Also support the async iterator pattern
|
|
412
|
+
if (stream && stream[Symbol.asyncIterator]) {
|
|
413
|
+
const origIterator = stream[Symbol.asyncIterator].bind(stream);
|
|
414
|
+
const chunks = [];
|
|
415
|
+
stream[Symbol.asyncIterator] = function () {
|
|
416
|
+
const iter = origIterator();
|
|
417
|
+
return {
|
|
418
|
+
async next() {
|
|
419
|
+
const result = await iter.next();
|
|
420
|
+
if (!result.done) {
|
|
421
|
+
const event = result.value;
|
|
422
|
+
if (event.type === 'content_block_delta' && event.delta?.text) {
|
|
423
|
+
chunks.push(event.delta.text);
|
|
424
|
+
}
|
|
425
|
+
if (event.type === 'message_stop' || event.type === 'message_delta') {
|
|
426
|
+
if (event.usage) {
|
|
427
|
+
const outputText = chunks.join('');
|
|
428
|
+
writeLlmEvent({
|
|
429
|
+
kind: 'llm_call', provider: 'anthropic', model: params.model || 'unknown',
|
|
430
|
+
durationMs: round(performance.now() - startTime),
|
|
431
|
+
inputTokens: event.usage.input_tokens || 0,
|
|
432
|
+
outputTokens: event.usage.output_tokens || 0,
|
|
433
|
+
totalTokens: (event.usage.input_tokens || 0) + (event.usage.output_tokens || 0),
|
|
434
|
+
estimatedCostUsd: estimateCost(params.model || '', event.usage.input_tokens || 0, event.usage.output_tokens || 0),
|
|
435
|
+
stream: true, finishReason: 'end_turn',
|
|
436
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
437
|
+
systemPrompt: typeof params.system === 'string' ? truncate(params.system, 200) : undefined,
|
|
438
|
+
inputPreview: extractInputPreview(params.messages),
|
|
439
|
+
outputPreview: truncate(outputText),
|
|
440
|
+
messageCount: params.messages?.length || 0,
|
|
441
|
+
toolUse: hasToolUse(params), timestamp: Date.now(),
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return result;
|
|
447
|
+
},
|
|
448
|
+
return: iter.return?.bind(iter),
|
|
449
|
+
throw: iter.throw?.bind(iter),
|
|
450
|
+
};
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
return stream;
|
|
454
|
+
}
|
|
455
|
+
function captureAnthropicResponse(params, response, startTime, debug) {
|
|
456
|
+
const usage = response.usage || {};
|
|
457
|
+
const outputText = response.content?.map((c) => c.text || '').join('') || '';
|
|
458
|
+
const event = {
|
|
459
|
+
kind: 'llm_call', provider: 'anthropic', model: response.model || params.model || 'unknown',
|
|
460
|
+
durationMs: round(performance.now() - startTime),
|
|
461
|
+
inputTokens: usage.input_tokens || 0, outputTokens: usage.output_tokens || 0,
|
|
462
|
+
totalTokens: (usage.input_tokens || 0) + (usage.output_tokens || 0),
|
|
463
|
+
estimatedCostUsd: estimateCost(response.model || params.model || '', usage.input_tokens || 0, usage.output_tokens || 0),
|
|
464
|
+
stream: false, finishReason: response.stop_reason || 'unknown',
|
|
465
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
466
|
+
systemPrompt: typeof params.system === 'string' ? truncate(params.system, 200) : undefined,
|
|
467
|
+
inputPreview: extractInputPreview(params.messages),
|
|
468
|
+
outputPreview: truncate(outputText),
|
|
469
|
+
messageCount: params.messages?.length || 0,
|
|
470
|
+
toolUse: hasToolUse(params) || response.content?.some((c) => c.type === 'tool_use'),
|
|
471
|
+
timestamp: Date.now(),
|
|
472
|
+
};
|
|
473
|
+
writeLlmEvent(event);
|
|
474
|
+
if (debug)
|
|
475
|
+
console.log(`[trickle/llm] Anthropic: ${event.model} (${event.totalTokens} tokens, ${event.durationMs}ms)`);
|
|
476
|
+
}
|
|
477
|
+
function captureAnthropicError(params, err, startTime, debug) {
|
|
478
|
+
writeLlmEvent({
|
|
479
|
+
kind: 'llm_call', provider: 'anthropic', model: params.model || 'unknown',
|
|
480
|
+
durationMs: round(performance.now() - startTime),
|
|
481
|
+
inputTokens: 0, outputTokens: 0, totalTokens: 0, estimatedCostUsd: 0,
|
|
482
|
+
stream: !!params.stream, finishReason: 'error',
|
|
483
|
+
temperature: params.temperature, maxTokens: params.max_tokens,
|
|
484
|
+
systemPrompt: typeof params.system === 'string' ? truncate(params.system, 200) : undefined,
|
|
485
|
+
inputPreview: extractInputPreview(params.messages),
|
|
486
|
+
outputPreview: '', messageCount: params.messages?.length || 0,
|
|
487
|
+
toolUse: hasToolUse(params), timestamp: Date.now(),
|
|
488
|
+
error: truncate(err?.message || String(err), 200),
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
// ────────────────────────────────────────────────────
|
|
492
|
+
// Helpers
|
|
493
|
+
// ────────────────────────────────────────────────────
|
|
494
|
+
function round(n) {
|
|
495
|
+
return Math.round(n * 100) / 100;
|
|
496
|
+
}
|
|
497
|
+
function getattr(obj, key) {
|
|
498
|
+
try {
|
|
499
|
+
return !!obj[key];
|
|
500
|
+
}
|
|
501
|
+
catch {
|
|
502
|
+
return false;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
function setattr(obj, key, val) {
|
|
506
|
+
try {
|
|
507
|
+
obj[key] = val;
|
|
508
|
+
}
|
|
509
|
+
catch { }
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Initialize the LLM observer — clears previous data file.
|
|
513
|
+
*/
|
|
514
|
+
function initLlmObserver() {
|
|
515
|
+
const dir = process.env.TRICKLE_LOCAL_DIR || path.join(process.cwd(), '.trickle');
|
|
516
|
+
try {
|
|
517
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
518
|
+
}
|
|
519
|
+
catch { }
|
|
520
|
+
llmFile = path.join(dir, 'llm.jsonl');
|
|
521
|
+
try {
|
|
522
|
+
fs.writeFileSync(llmFile, '');
|
|
523
|
+
}
|
|
524
|
+
catch { }
|
|
525
|
+
eventCount = 0;
|
|
526
|
+
}
|
package/dist/observe-register.js
CHANGED
|
@@ -41,6 +41,7 @@ const fetch_observer_1 = require("./fetch-observer");
|
|
|
41
41
|
const express_1 = require("./express");
|
|
42
42
|
const trace_var_1 = require("./trace-var");
|
|
43
43
|
const call_trace_1 = require("./call-trace");
|
|
44
|
+
const llm_observer_1 = require("./llm-observer");
|
|
44
45
|
const vite_plugin_1 = require("./vite-plugin");
|
|
45
46
|
// ── Source map support ──
|
|
46
47
|
// Lightweight VLQ decoder for mapping compiled JS lines back to original TS lines
|
|
@@ -1239,6 +1240,8 @@ if (enabled) {
|
|
|
1239
1240
|
}
|
|
1240
1241
|
// ── Hook 0b2: Initialize call trace ──
|
|
1241
1242
|
(0, call_trace_1.initCallTrace)();
|
|
1243
|
+
// ── Hook 0b3: Initialize LLM observer ──
|
|
1244
|
+
(0, llm_observer_1.initLlmObserver)();
|
|
1242
1245
|
// ── Hook 0c: Capture environment snapshot ──
|
|
1243
1246
|
try {
|
|
1244
1247
|
const envDir = process.env.TRICKLE_LOCAL_DIR || path_1.default.join(process.cwd(), '.trickle');
|
|
@@ -1531,6 +1534,24 @@ if (enabled) {
|
|
|
1531
1534
|
}
|
|
1532
1535
|
catch { /* not critical */ }
|
|
1533
1536
|
}
|
|
1537
|
+
// OpenAI SDK
|
|
1538
|
+
if (request === 'openai' && !expressPatched.has('openai')) {
|
|
1539
|
+
expressPatched.add('openai');
|
|
1540
|
+
try {
|
|
1541
|
+
const { patchOpenAI } = require(path_1.default.join(__dirname, 'llm-observer.js'));
|
|
1542
|
+
patchOpenAI(exports, debug);
|
|
1543
|
+
}
|
|
1544
|
+
catch { /* not critical */ }
|
|
1545
|
+
}
|
|
1546
|
+
// Anthropic SDK
|
|
1547
|
+
if ((request === '@anthropic-ai/sdk' || request === 'anthropic') && !expressPatched.has('anthropic')) {
|
|
1548
|
+
expressPatched.add('anthropic');
|
|
1549
|
+
try {
|
|
1550
|
+
const { patchAnthropic } = require(path_1.default.join(__dirname, 'llm-observer.js'));
|
|
1551
|
+
patchAnthropic(exports, debug);
|
|
1552
|
+
}
|
|
1553
|
+
catch { /* not critical */ }
|
|
1554
|
+
}
|
|
1534
1555
|
// Resolve to absolute path for dedup — do this FIRST since bundlers like
|
|
1535
1556
|
// tsx/esbuild may use path aliases (e.g., @config/env) that don't start
|
|
1536
1557
|
// with './' or '/'. We need the resolved path to decide if it's user code.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|