weflayr 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +1 -1
- package/package.json +1 -1
- package/src/instrumentation/anthropic/get_streaming_stop_reason_telemetry.js +121 -0
- package/src/instrumentation/anthropic/get_tool_definitions_telemetry.js +50 -0
- package/src/instrumentation/anthropic/set_telemetry_attributes.js +13 -3
- package/src/instrumentation/bedrock/get_input_message_content_telemetry.js +94 -0
- package/src/instrumentation/bedrock/get_streaming_output_message_telemetry.js +87 -0
- package/src/instrumentation/bedrock/set_telemetry_attributes.js +12 -3
- package/src/instrumentation/cohere/get_embeddings_telemetry.js +10 -0
- package/src/instrumentation/google/get_audio_telemetry.js +1 -2
- package/src/instrumentation/google/get_cache_telemetry.js +1 -3
- package/src/instrumentation/google/get_embeddings_telemetry.js +22 -1
- package/src/instrumentation/openai/get_embeddings_telemetry.js +12 -0
- package/src/otel/exporter.js +1 -6
- package/src/otel/span-processor.js +30 -1
- package/src/setup.js +1 -2
- package/tests/index.test.js +102 -0
- package/tests/instrumentation/anthropic/get_cache_and_reasoning_telemetry.test.js +33 -0
- package/tests/instrumentation/anthropic/get_streaming_stop_reason_telemetry.test.js +117 -0
- package/tests/instrumentation/anthropic/get_tool_definitions_telemetry.test.js +82 -0
- package/tests/instrumentation/bedrock/get_input_message_content_telemetry.test.js +89 -0
- package/tests/instrumentation/bedrock/get_streaming_output_message_telemetry.test.js +87 -0
- package/tests/instrumentation/cohere/get_embeddings_telemetry.test.js +3 -0
- package/tests/instrumentation/elevenlabs/full_instrumentation.test.js +18 -4
- package/tests/instrumentation/google/get_embeddings_telemetry.test.js +3 -0
- package/tests/instrumentation/google/set_telemetry_attributes.test.js +33 -1
- package/tests/instrumentation/openai/get_embeddings_telemetry.test.js +19 -0
- package/tests/instrumentation/openai/get_streaming_usage.test.js +29 -0
package/tests/index.test.js
CHANGED
|
@@ -184,6 +184,54 @@ test('buildExporter strips trailing slash on baseUrl', () => {
|
|
|
184
184
|
assert.equal(_exporterParams(exporter).url, 'https://api.weflayr.com/');
|
|
185
185
|
});
|
|
186
186
|
|
|
187
|
+
test('buildExporter warns on every export error when warnOnExportError is true', async () => {
|
|
188
|
+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
|
|
189
|
+
const origExport = OTLPTraceExporter.prototype.export;
|
|
190
|
+
OTLPTraceExporter.prototype.export = function (spans, cb) {
|
|
191
|
+
cb({ code: 1, error: { code: 401, message: 'Unauthorized' } });
|
|
192
|
+
};
|
|
193
|
+
const warnings = [];
|
|
194
|
+
const origWarn = console.warn;
|
|
195
|
+
console.warn = (...args) => warnings.push(args.join(' '));
|
|
196
|
+
try {
|
|
197
|
+
const exporter = buildExporter({
|
|
198
|
+
baseUrl: 'https://api.weflayr.com',
|
|
199
|
+
apiKey: 'bad',
|
|
200
|
+
warnOnExportError: true,
|
|
201
|
+
});
|
|
202
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
203
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
204
|
+
assert.equal(warnings.filter((w) => w.includes('[weflayr]')).length, 2);
|
|
205
|
+
} finally {
|
|
206
|
+
OTLPTraceExporter.prototype.export = origExport;
|
|
207
|
+
console.warn = origWarn;
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test('buildExporter does not warn when warnOnExportError is false', async () => {
|
|
212
|
+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
|
|
213
|
+
const origExport = OTLPTraceExporter.prototype.export;
|
|
214
|
+
OTLPTraceExporter.prototype.export = function (spans, cb) {
|
|
215
|
+
cb({ code: 1, error: { code: 401, message: 'Unauthorized' } });
|
|
216
|
+
};
|
|
217
|
+
const warnings = [];
|
|
218
|
+
const origWarn = console.warn;
|
|
219
|
+
console.warn = (...args) => warnings.push(args.join(' '));
|
|
220
|
+
try {
|
|
221
|
+
const exporter = buildExporter({
|
|
222
|
+
baseUrl: 'https://api.weflayr.com',
|
|
223
|
+
apiKey: 'bad',
|
|
224
|
+
warnOnExportError: false,
|
|
225
|
+
});
|
|
226
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
227
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
228
|
+
assert.equal(warnings.filter((w) => w.includes('[weflayr]')).length, 0);
|
|
229
|
+
} finally {
|
|
230
|
+
OTLPTraceExporter.prototype.export = origExport;
|
|
231
|
+
console.warn = origWarn;
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
|
|
187
235
|
// ── span-filter ───────────────────────────────────────────────────────────────
|
|
188
236
|
|
|
189
237
|
test('isDefaultExportSpan accepts spans with gen_ai.* attributes', () => {
|
|
@@ -287,6 +335,60 @@ test('WeflayrSpanProcessor applies mask before export', async () => {
|
|
|
287
335
|
assert.equal(span.attributes['gen_ai.prompt'], '[REDACTED]');
|
|
288
336
|
});
|
|
289
337
|
|
|
338
|
+
test('WeflayrSpanProcessor strips message content when capture disabled', async () => {
|
|
339
|
+
// Instrumentors gate content on their own env vars (openllmetry captures by
|
|
340
|
+
// default), so the processor itself must drop message content when
|
|
341
|
+
// captureMessageContent=false recorded "false" in the env var.
|
|
342
|
+
process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = 'false';
|
|
343
|
+
try {
|
|
344
|
+
const exporter = makeExporterStub();
|
|
345
|
+
const proc = new WeflayrSpanProcessor({ exporter, exportMode: 'immediate' });
|
|
346
|
+
const span = makeFakeSpan({
|
|
347
|
+
attributes: {
|
|
348
|
+
'gen_ai.system': 'openai',
|
|
349
|
+
'gen_ai.input.messages': '[{"role": "user"}]',
|
|
350
|
+
'gen_ai.output.messages': '[{"role": "assistant"}]',
|
|
351
|
+
'gen_ai.system_instructions': '[{"type": "text"}]',
|
|
352
|
+
'gen_ai.prompt.0.content': 'Hi Noe',
|
|
353
|
+
'gen_ai.completion.0.content': 'Hello!',
|
|
354
|
+
'gen_ai.usage.input_tokens': 12,
|
|
355
|
+
},
|
|
356
|
+
});
|
|
357
|
+
proc.onEnd(span);
|
|
358
|
+
await proc.forceFlush();
|
|
359
|
+
assert.equal(span.attributes['gen_ai.system'], 'openai');
|
|
360
|
+
assert.equal(span.attributes['gen_ai.usage.input_tokens'], 12);
|
|
361
|
+
for (const key of Object.keys(span.attributes)) {
|
|
362
|
+
assert.ok(!key.startsWith('gen_ai.prompt'), `unexpected ${key}`);
|
|
363
|
+
assert.ok(!key.startsWith('gen_ai.completion'), `unexpected ${key}`);
|
|
364
|
+
}
|
|
365
|
+
assert.ok(!('gen_ai.input.messages' in span.attributes));
|
|
366
|
+
assert.ok(!('gen_ai.output.messages' in span.attributes));
|
|
367
|
+
assert.ok(!('gen_ai.system_instructions' in span.attributes));
|
|
368
|
+
} finally {
|
|
369
|
+
delete process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT;
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
test('WeflayrSpanProcessor keeps message content when capture enabled', async () => {
|
|
374
|
+
process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = 'true';
|
|
375
|
+
try {
|
|
376
|
+
const exporter = makeExporterStub();
|
|
377
|
+
const proc = new WeflayrSpanProcessor({ exporter, exportMode: 'immediate' });
|
|
378
|
+
const span = makeFakeSpan({
|
|
379
|
+
attributes: {
|
|
380
|
+
'gen_ai.system': 'openai',
|
|
381
|
+
'gen_ai.input.messages': '[{"role": "user"}]',
|
|
382
|
+
},
|
|
383
|
+
});
|
|
384
|
+
proc.onEnd(span);
|
|
385
|
+
await proc.forceFlush();
|
|
386
|
+
assert.equal(span.attributes['gen_ai.input.messages'], '[{"role": "user"}]');
|
|
387
|
+
} finally {
|
|
388
|
+
delete process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT;
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
|
|
290
392
|
// ── propagateMetadata ────────────────────────────────────────────────────────
|
|
291
393
|
|
|
292
394
|
function _makeMetadataHarness() {
|
|
@@ -118,6 +118,39 @@ test('stamps cache + raw input from the streamed message_start usage', async ()
|
|
|
118
118
|
assert.equal(span.attrs['gen_ai.usage.input_tokens'], 10);
|
|
119
119
|
});
|
|
120
120
|
|
|
121
|
+
test('re-yields every stream chunk unchanged — no telemetry collision with the payload', async () => {
|
|
122
|
+
const emitted = [
|
|
123
|
+
{
|
|
124
|
+
type: 'message_start',
|
|
125
|
+
message: { usage: { input_tokens: 10, cache_read_input_tokens: 5270 } },
|
|
126
|
+
},
|
|
127
|
+
{ type: 'content_block_delta', delta: { text: 'O' } },
|
|
128
|
+
{ type: 'content_block_delta', delta: { text: 'K' } },
|
|
129
|
+
];
|
|
130
|
+
const instrumentation = {
|
|
131
|
+
_endSpan() {},
|
|
132
|
+
_streamingWrapPromise() {
|
|
133
|
+
return (async function* () {
|
|
134
|
+
for (const chunk of emitted) yield chunk;
|
|
135
|
+
})();
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
139
|
+
|
|
140
|
+
const span = fakeSpan();
|
|
141
|
+
const collected = [];
|
|
142
|
+
for await (const chunk of instrumentation._streamingWrapPromise('client', 'mod', { span })) {
|
|
143
|
+
collected.push(chunk);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Same count, order, and object identity: observeCache re-yields each chunk by
|
|
147
|
+
// reference without copying or mutating it.
|
|
148
|
+
assert.equal(collected.length, emitted.length);
|
|
149
|
+
collected.forEach((chunk, index) => assert.equal(chunk, emitted[index]));
|
|
150
|
+
// …while still stamping cache + raw input off the message_start chunk.
|
|
151
|
+
assert.equal(span.attrs['gen_ai.usage.input_tokens'], 10);
|
|
152
|
+
});
|
|
153
|
+
|
|
121
154
|
test('is a no-op when the instrumentor has no _endSpan', () => {
|
|
122
155
|
const instrumentation = {};
|
|
123
156
|
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Anthropic streamed stop-reason complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed the way the real one works:
|
|
6
|
+
// `_streamingWrapPromise` returns `new moduleExports.APIPromise(...)` whose
|
|
7
|
+
// parse step wraps the SDK stream with an accumulating iterator that calls
|
|
8
|
+
// `_endSpan` when exhausted — so the test exercises the complement's APIPromise
|
|
9
|
+
// interposition end to end.
|
|
10
|
+
|
|
11
|
+
const test = require('node:test');
|
|
12
|
+
const assert = require('node:assert/strict');
|
|
13
|
+
|
|
14
|
+
const {
|
|
15
|
+
AnthropicStreamingStopReasonInstrumentor,
|
|
16
|
+
} = require('../../../src/instrumentation/anthropic/get_streaming_stop_reason_telemetry');
|
|
17
|
+
|
|
18
|
+
class StubStream {
|
|
19
|
+
constructor(iteratorFactory, controller) {
|
|
20
|
+
this._iteratorFactory = iteratorFactory;
|
|
21
|
+
this.controller = controller;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
[Symbol.asyncIterator]() {
|
|
25
|
+
return this._iteratorFactory();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
class StubAPIPromise {
|
|
30
|
+
constructor(client, responsePromise, parseResponse) {
|
|
31
|
+
this.parsed = parseResponse(client, responsePromise);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function makeStubInstrumentation() {
|
|
36
|
+
const finalized = [];
|
|
37
|
+
return {
|
|
38
|
+
finalized,
|
|
39
|
+
_streamingWrapPromise(client, moduleExports, { span, promise }) {
|
|
40
|
+
const self = this;
|
|
41
|
+
return new moduleExports.APIPromise(
|
|
42
|
+
client,
|
|
43
|
+
promise,
|
|
44
|
+
async (parseClient, realStreamPromise) => {
|
|
45
|
+
const realStream = await realStreamPromise;
|
|
46
|
+
// Mimics traceloop: re-wrap the stream with an accumulating iterator
|
|
47
|
+
// that finalizes the span once exhausted.
|
|
48
|
+
return new realStream.constructor(
|
|
49
|
+
() =>
|
|
50
|
+
(async function* accumulate() {
|
|
51
|
+
const result = { stop_reason: null };
|
|
52
|
+
for await (const chunk of realStream) yield chunk;
|
|
53
|
+
self._endSpan({ span, result });
|
|
54
|
+
})(),
|
|
55
|
+
realStream.controller
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
);
|
|
59
|
+
},
|
|
60
|
+
_endSpan(args) {
|
|
61
|
+
finalized.push(args.result);
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function chunkStream() {
|
|
67
|
+
return new StubStream(async function* chunks() {
|
|
68
|
+
yield { type: 'message_start', message: { usage: { input_tokens: 5 } } };
|
|
69
|
+
yield { type: 'content_block_delta', delta: { type: 'text_delta', text: 'Hello' } };
|
|
70
|
+
yield {
|
|
71
|
+
type: 'message_delta',
|
|
72
|
+
delta: { stop_reason: 'end_turn' },
|
|
73
|
+
usage: { output_tokens: 3 },
|
|
74
|
+
};
|
|
75
|
+
yield { type: 'message_stop' };
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
test('injects the streamed stop reason into the result before endSpan', async () => {
|
|
80
|
+
const instrumentation = makeStubInstrumentation();
|
|
81
|
+
new AnthropicStreamingStopReasonInstrumentor().instrument(instrumentation);
|
|
82
|
+
|
|
83
|
+
const span = { name: 'anthropic.chat' };
|
|
84
|
+
const apiPromise = instrumentation._streamingWrapPromise(
|
|
85
|
+
{},
|
|
86
|
+
{ APIPromise: StubAPIPromise },
|
|
87
|
+
{ span, type: 'chat', promise: Promise.resolve(chunkStream()) }
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
const stream = await apiPromise.parsed;
|
|
91
|
+
const seen = [];
|
|
92
|
+
for await (const chunk of stream) seen.push(chunk.type);
|
|
93
|
+
|
|
94
|
+
assert.deepEqual(seen, ['message_start', 'content_block_delta', 'message_delta', 'message_stop']);
|
|
95
|
+
assert.equal(instrumentation.finalized.length, 1);
|
|
96
|
+
assert.equal(instrumentation.finalized[0].stop_reason, 'end_turn');
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('leaves an already-set stop reason untouched', () => {
|
|
100
|
+
const instrumentation = makeStubInstrumentation();
|
|
101
|
+
new AnthropicStreamingStopReasonInstrumentor().instrument(instrumentation);
|
|
102
|
+
|
|
103
|
+
const span = { name: 'anthropic.chat' };
|
|
104
|
+
instrumentation._endSpan({ span, result: { stop_reason: 'max_tokens' } });
|
|
105
|
+
assert.equal(instrumentation.finalized[0].stop_reason, 'max_tokens');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('uninstrument restores the original methods', () => {
|
|
109
|
+
const instrumentation = makeStubInstrumentation();
|
|
110
|
+
const streamOriginal = instrumentation._streamingWrapPromise;
|
|
111
|
+
const endSpanOriginal = instrumentation._endSpan;
|
|
112
|
+
const instrumentor = new AnthropicStreamingStopReasonInstrumentor();
|
|
113
|
+
instrumentor.instrument(instrumentation);
|
|
114
|
+
instrumentor.uninstrument();
|
|
115
|
+
assert.equal(instrumentation._streamingWrapPromise, streamOriginal);
|
|
116
|
+
assert.equal(instrumentation._endSpan, endSpanOriginal);
|
|
117
|
+
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Anthropic tool-definitions complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed with a `startSpan` that
|
|
6
|
+
// returns a fake span, so the wrap can be exercised without the real SDK.
|
|
7
|
+
|
|
8
|
+
const test = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
|
|
11
|
+
const {
|
|
12
|
+
AnthropicToolDefinitionsInstrumentor,
|
|
13
|
+
} = require('../../../src/instrumentation/anthropic/get_tool_definitions_telemetry');
|
|
14
|
+
|
|
15
|
+
function makeFakeSpan() {
|
|
16
|
+
const attributes = {};
|
|
17
|
+
return {
|
|
18
|
+
attributes,
|
|
19
|
+
setAttribute(key, value) {
|
|
20
|
+
attributes[key] = value;
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function makeStubInstrumentation() {
|
|
26
|
+
return {
|
|
27
|
+
startSpan(request) {
|
|
28
|
+
const span = makeFakeSpan();
|
|
29
|
+
span.attributes['gen_ai.request.model'] = request.params.model;
|
|
30
|
+
return span;
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const WEATHER_TOOL = {
|
|
36
|
+
name: 'get_weather',
|
|
37
|
+
description: 'Get the current weather for a city.',
|
|
38
|
+
input_schema: { type: 'object', properties: { city: { type: 'string' } } },
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
test('stamps tool definitions from the request params', () => {
|
|
42
|
+
const instrumentation = makeStubInstrumentation();
|
|
43
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
44
|
+
instrumentor.instrument(instrumentation);
|
|
45
|
+
|
|
46
|
+
const span = instrumentation.startSpan({
|
|
47
|
+
type: 'chat',
|
|
48
|
+
params: { model: 'claude-haiku-4-5', tools: [WEATHER_TOOL] },
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
assert.equal(span.attributes['gen_ai.request.model'], 'claude-haiku-4-5');
|
|
52
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.tool.definitions']), [WEATHER_TOOL]);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('no attribute when the request has no tools', () => {
|
|
56
|
+
const instrumentation = makeStubInstrumentation();
|
|
57
|
+
new AnthropicToolDefinitionsInstrumentor().instrument(instrumentation);
|
|
58
|
+
|
|
59
|
+
const span = instrumentation.startSpan({
|
|
60
|
+
type: 'chat',
|
|
61
|
+
params: { model: 'claude-haiku-4-5' },
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
assert.ok(!('gen_ai.tool.definitions' in span.attributes));
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('uninstrument restores the original startSpan', () => {
|
|
68
|
+
const instrumentation = makeStubInstrumentation();
|
|
69
|
+
const original = instrumentation.startSpan;
|
|
70
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
71
|
+
instrumentor.instrument(instrumentation);
|
|
72
|
+
assert.notEqual(instrumentation.startSpan, original);
|
|
73
|
+
instrumentor.uninstrument();
|
|
74
|
+
assert.equal(instrumentation.startSpan, original);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('is a no-op when the instrumentor has no startSpan', () => {
|
|
78
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
79
|
+
instrumentor.instrument({});
|
|
80
|
+
instrumentor.instrument(undefined);
|
|
81
|
+
instrumentor.uninstrument();
|
|
82
|
+
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Bedrock input-message-content complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed with a `_startSpan` that
|
|
6
|
+
// returns a fake span — with or without instrumentor-set input messages — so
|
|
7
|
+
// the wrap's "only when missing" behaviour can be exercised.
|
|
8
|
+
|
|
9
|
+
const test = require('node:test');
|
|
10
|
+
const assert = require('node:assert/strict');
|
|
11
|
+
|
|
12
|
+
const {
|
|
13
|
+
BedrockInputMessageContentInstrumentor,
|
|
14
|
+
} = require('../../../src/instrumentation/bedrock/get_input_message_content_telemetry');
|
|
15
|
+
|
|
16
|
+
function makeFakeSpan(attributes = {}) {
|
|
17
|
+
return {
|
|
18
|
+
attributes,
|
|
19
|
+
setAttribute(key, value) {
|
|
20
|
+
attributes[key] = value;
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function makeStubInstrumentation(spanAttributes) {
|
|
26
|
+
return {
|
|
27
|
+
_startSpan() {
|
|
28
|
+
return makeFakeSpan(spanAttributes);
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function request(body) {
|
|
34
|
+
return { params: { input: { modelId: 'model-that-does-not-exist', body } } };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
test('stamps input messages when the instrumentor set none', () => {
|
|
38
|
+
const instrumentation = makeStubInstrumentation({});
|
|
39
|
+
new BedrockInputMessageContentInstrumentor().instrument(instrumentation);
|
|
40
|
+
|
|
41
|
+
const span = instrumentation._startSpan(
|
|
42
|
+
request(JSON.stringify({ messages: [{ role: 'user', content: [{ text: 'Hello' }] }] }))
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.input.messages']), [
|
|
46
|
+
{ role: 'user', parts: [{ type: 'text', content: 'Hello' }] },
|
|
47
|
+
]);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('supports the flat inputText schema', () => {
|
|
51
|
+
const instrumentation = makeStubInstrumentation({});
|
|
52
|
+
new BedrockInputMessageContentInstrumentor().instrument(instrumentation);
|
|
53
|
+
|
|
54
|
+
const span = instrumentation._startSpan(request(JSON.stringify({ inputText: 'Hello' })));
|
|
55
|
+
|
|
56
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.input.messages']), [
|
|
57
|
+
{ role: 'user', parts: [{ type: 'text', content: 'Hello' }] },
|
|
58
|
+
]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('leaves instrumentor-set input messages untouched', () => {
|
|
62
|
+
const existing = '[{"role":"user","parts":[{"type":"text","content":"from-traceloop"}]}]';
|
|
63
|
+
const instrumentation = makeStubInstrumentation({ 'gen_ai.input.messages': existing });
|
|
64
|
+
new BedrockInputMessageContentInstrumentor().instrument(instrumentation);
|
|
65
|
+
|
|
66
|
+
const span = instrumentation._startSpan(
|
|
67
|
+
request(JSON.stringify({ messages: [{ role: 'user', content: [{ text: 'other' }] }] }))
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
assert.equal(span.attributes['gen_ai.input.messages'], existing);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test('no attribute for an unparseable or empty body', () => {
|
|
74
|
+
const instrumentation = makeStubInstrumentation({});
|
|
75
|
+
new BedrockInputMessageContentInstrumentor().instrument(instrumentation);
|
|
76
|
+
|
|
77
|
+
assert.ok(
|
|
78
|
+
!('gen_ai.input.messages' in instrumentation._startSpan(request('not-json')).attributes)
|
|
79
|
+
);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test('uninstrument restores the original startSpan', () => {
|
|
83
|
+
const instrumentation = makeStubInstrumentation({});
|
|
84
|
+
const original = instrumentation._startSpan;
|
|
85
|
+
const instrumentor = new BedrockInputMessageContentInstrumentor();
|
|
86
|
+
instrumentor.instrument(instrumentation);
|
|
87
|
+
instrumentor.uninstrument();
|
|
88
|
+
assert.equal(instrumentation._startSpan, original);
|
|
89
|
+
});
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Bedrock streamed-output message complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed with a
|
|
6
|
+
// `_handleNovaStreamingMetadata` no-op; chunks are replayed through the wrap the
|
|
7
|
+
// way the instrumentation's stream loop calls it.
|
|
8
|
+
|
|
9
|
+
const test = require('node:test');
|
|
10
|
+
const assert = require('node:assert/strict');
|
|
11
|
+
|
|
12
|
+
const {
|
|
13
|
+
BedrockStreamingOutputMessageInstrumentor,
|
|
14
|
+
} = require('../../../src/instrumentation/bedrock/get_streaming_output_message_telemetry');
|
|
15
|
+
|
|
16
|
+
function makeFakeSpan() {
|
|
17
|
+
const attributes = {};
|
|
18
|
+
return {
|
|
19
|
+
attributes,
|
|
20
|
+
setAttribute(key, value) {
|
|
21
|
+
attributes[key] = value;
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function instrumented() {
|
|
27
|
+
const instrumentation = { _handleNovaStreamingMetadata() {} };
|
|
28
|
+
const instrumentor = new BedrockStreamingOutputMessageInstrumentor();
|
|
29
|
+
instrumentor.instrument(instrumentation);
|
|
30
|
+
return { instrumentation, instrumentor };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
test('re-stamps the output message from accumulated stream chunks', () => {
|
|
34
|
+
const { instrumentation } = instrumented();
|
|
35
|
+
const span = makeFakeSpan();
|
|
36
|
+
|
|
37
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
38
|
+
contentBlockDelta: { delta: { text: 'Hel' } },
|
|
39
|
+
});
|
|
40
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
41
|
+
contentBlockDelta: { delta: { text: 'lo!' } },
|
|
42
|
+
});
|
|
43
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
44
|
+
messageStop: { stopReason: 'end_turn' },
|
|
45
|
+
});
|
|
46
|
+
assert.ok(!('gen_ai.output.messages' in span.attributes), 'set only on the metadata chunk');
|
|
47
|
+
|
|
48
|
+
instrumentation._handleNovaStreamingMetadata(span, { metadata: { usage: {} } });
|
|
49
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.output.messages']), [
|
|
50
|
+
{
|
|
51
|
+
role: 'assistant',
|
|
52
|
+
parts: [{ type: 'text', content: 'Hello!' }],
|
|
53
|
+
finish_reason: 'stop',
|
|
54
|
+
},
|
|
55
|
+
]);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('maps max_tokens to length', () => {
|
|
59
|
+
const { instrumentation } = instrumented();
|
|
60
|
+
const span = makeFakeSpan();
|
|
61
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
62
|
+
contentBlockDelta: { delta: { text: 'OK' } },
|
|
63
|
+
});
|
|
64
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
65
|
+
messageStop: { stopReason: 'max_tokens' },
|
|
66
|
+
});
|
|
67
|
+
instrumentation._handleNovaStreamingMetadata(span, { metadata: {} });
|
|
68
|
+
assert.equal(JSON.parse(span.attributes['gen_ai.output.messages'])[0].finish_reason, 'length');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('no attribute when no text was streamed', () => {
|
|
72
|
+
const { instrumentation } = instrumented();
|
|
73
|
+
const span = makeFakeSpan();
|
|
74
|
+
instrumentation._handleNovaStreamingMetadata(span, {
|
|
75
|
+
messageStop: { stopReason: 'end_turn' },
|
|
76
|
+
});
|
|
77
|
+
instrumentation._handleNovaStreamingMetadata(span, { metadata: {} });
|
|
78
|
+
assert.ok(!('gen_ai.output.messages' in span.attributes));
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('uninstrument restores the original handler', () => {
|
|
82
|
+
const { instrumentation, instrumentor } = instrumented();
|
|
83
|
+
instrumentor.uninstrument();
|
|
84
|
+
const span = makeFakeSpan();
|
|
85
|
+
instrumentation._handleNovaStreamingMetadata(span, { metadata: {} });
|
|
86
|
+
assert.deepEqual(span.attributes, {});
|
|
87
|
+
});
|
|
@@ -84,6 +84,9 @@ test('v1 embed emits genai span with input tokens', async () => {
|
|
|
84
84
|
assert.equal(span.attributes['gen_ai.operation.name'], 'embeddings');
|
|
85
85
|
assert.equal(span.attributes['gen_ai.request.model'], 'embed-english-v3.0');
|
|
86
86
|
assert.equal(span.attributes['gen_ai.usage.input_tokens'], 8);
|
|
87
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.input.messages']), [
|
|
88
|
+
{ role: 'user', parts: [{ type: 'text', content: 'hi' }] },
|
|
89
|
+
]);
|
|
87
90
|
assert.equal(span.status.code, SpanStatusCode.UNSET);
|
|
88
91
|
} finally {
|
|
89
92
|
teardown();
|
|
@@ -35,6 +35,12 @@ trace.setGlobalTracerProvider(
|
|
|
35
35
|
new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
|
|
36
36
|
);
|
|
37
37
|
|
|
38
|
+
// The exact chunk objects the streaming stubs yield. Shared so the passthrough
|
|
39
|
+
// assertions can compare object identity — proof our proxies re-emit the user's
|
|
40
|
+
// chunks without copying or mutating them.
|
|
41
|
+
const TTS_BYTE_CHUNKS = [Buffer.from('a'), Buffer.from('b')];
|
|
42
|
+
const TTS_OBJECT_CHUNKS = [{ audioBase64: 'eA==' }, { audioBase64: 'YmI=' }];
|
|
43
|
+
|
|
38
44
|
function readableOf(chunks) {
|
|
39
45
|
let index = 0;
|
|
40
46
|
return new ReadableStream({
|
|
@@ -64,7 +70,7 @@ function asyncIterableOf(chunks) {
|
|
|
64
70
|
class StubTextToSpeechClient {
|
|
65
71
|
convert(_voiceId, request = {}) {
|
|
66
72
|
if (request.text === 'boom') return Promise.resolve(failingReadable(new Error('api error')));
|
|
67
|
-
return Promise.resolve(readableOf(
|
|
73
|
+
return Promise.resolve(readableOf(TTS_BYTE_CHUNKS));
|
|
68
74
|
}
|
|
69
75
|
|
|
70
76
|
stream(_voiceId, _request = {}) {
|
|
@@ -72,7 +78,7 @@ class StubTextToSpeechClient {
|
|
|
72
78
|
}
|
|
73
79
|
|
|
74
80
|
streamWithTimestamps(_voiceId, _request = {}) {
|
|
75
|
-
return Promise.resolve(asyncIterableOf(
|
|
81
|
+
return Promise.resolve(asyncIterableOf(TTS_OBJECT_CHUNKS));
|
|
76
82
|
}
|
|
77
83
|
|
|
78
84
|
convertWithTimestamps(_voiceId, _request = {}) {
|
|
@@ -136,7 +142,11 @@ test('convert emits genai span with char count', async () => {
|
|
|
136
142
|
try {
|
|
137
143
|
const client = new StubTextToSpeechClient();
|
|
138
144
|
const stream = await client.convert('voice', { text: 'hello world', modelId: 'eleven_v2' });
|
|
139
|
-
|
|
145
|
+
// Same count, order, and object identity: the byte-stream proxy re-emits each
|
|
146
|
+
// chunk by reference — no telemetry collision with the caller's audio.
|
|
147
|
+
const collected = await drain(stream);
|
|
148
|
+
assert.equal(collected.length, TTS_BYTE_CHUNKS.length);
|
|
149
|
+
collected.forEach((chunk, index) => assert.equal(chunk, TTS_BYTE_CHUNKS[index]));
|
|
140
150
|
|
|
141
151
|
const [span] = exporter.getFinishedSpans();
|
|
142
152
|
assert.equal(span.name, 'text_to_speech');
|
|
@@ -186,7 +196,11 @@ test('stream with timestamps emits span with char count', async () => {
|
|
|
186
196
|
try {
|
|
187
197
|
const client = new StubTextToSpeechClient();
|
|
188
198
|
const stream = await client.streamWithTimestamps('voice', { text: 'hello', modelId: 'm' });
|
|
189
|
-
|
|
199
|
+
// Same count, order, and object identity: the object-stream proxy re-yields
|
|
200
|
+
// each chunk by reference — no telemetry collision with the caller's payload.
|
|
201
|
+
const collected = await drainIterable(stream);
|
|
202
|
+
assert.equal(collected.length, TTS_OBJECT_CHUNKS.length);
|
|
203
|
+
collected.forEach((chunk, index) => assert.equal(chunk, TTS_OBJECT_CHUNKS[index]));
|
|
190
204
|
|
|
191
205
|
const [span] = exporter.getFinishedSpans();
|
|
192
206
|
assert.equal(span.name, 'text_to_speech');
|
|
@@ -60,6 +60,9 @@ test('embeddings emits genai span with character count and stripped model', asyn
|
|
|
60
60
|
// The "models/" prefix is stripped to match the chat path.
|
|
61
61
|
assert.equal(span.attributes['gen_ai.request.model'], 'text-embedding-004');
|
|
62
62
|
assert.equal(span.attributes['gen_ai.usage.input_characters'], 'hello world'.length);
|
|
63
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.input.messages']), [
|
|
64
|
+
{ role: 'user', parts: [{ type: 'text', content: 'hello world' }] },
|
|
65
|
+
]);
|
|
63
66
|
assert.equal(span.status.code, SpanStatusCode.UNSET);
|
|
64
67
|
} finally {
|
|
65
68
|
teardown();
|
|
@@ -25,6 +25,14 @@ const {
|
|
|
25
25
|
|
|
26
26
|
const GENAI_MODULE = '@google/genai';
|
|
27
27
|
|
|
28
|
+
// The exact chunk objects the streaming stub yields. Shared so the passthrough
|
|
29
|
+
// test can assert object identity — proof our wrapper re-yields the user's chunks
|
|
30
|
+
// without copying or mutating them. The last chunk carries the streamed usage.
|
|
31
|
+
const STREAM_CHUNKS = [
|
|
32
|
+
{ text: 'hello ' },
|
|
33
|
+
{ text: 'world', usageMetadata: { cachedContentTokenCount: 32 } },
|
|
34
|
+
];
|
|
35
|
+
|
|
28
36
|
const exporter = new InMemorySpanExporter();
|
|
29
37
|
trace.setGlobalTracerProvider(
|
|
30
38
|
new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
|
|
@@ -51,7 +59,7 @@ class StubModels {
|
|
|
51
59
|
generateContentStreamInternal() {
|
|
52
60
|
return Promise.resolve(
|
|
53
61
|
(async function* () {
|
|
54
|
-
|
|
62
|
+
for (const chunk of STREAM_CHUNKS) yield chunk;
|
|
55
63
|
})()
|
|
56
64
|
);
|
|
57
65
|
}
|
|
@@ -185,3 +193,27 @@ test('streaming sets provider only and uninstrument restores methods', async ()
|
|
|
185
193
|
const client = new StubModels();
|
|
186
194
|
assert.equal(typeof client.generateContentInternal, 'function');
|
|
187
195
|
});
|
|
196
|
+
|
|
197
|
+
test('streaming re-yields the user chunks unchanged and stamps cache off the stream', async () => {
|
|
198
|
+
const teardown = instrumented();
|
|
199
|
+
try {
|
|
200
|
+
const client = new StubModels();
|
|
201
|
+
client.apiClient = { isVertexAI: () => false };
|
|
202
|
+
const collected = [];
|
|
203
|
+
const span = await withActiveSpan(async () => {
|
|
204
|
+
const stream = await client.generateContentStreamInternal({ model: 'm' });
|
|
205
|
+
for await (const chunk of stream) collected.push(chunk);
|
|
206
|
+
});
|
|
207
|
+
// Same count, order, and object identity: observeCache re-yields each chunk
|
|
208
|
+
// by reference — no telemetry collision with the caller's payload.
|
|
209
|
+
assert.equal(collected.length, STREAM_CHUNKS.length);
|
|
210
|
+
collected.forEach((chunk, index) => {
|
|
211
|
+
assert.equal(chunk, STREAM_CHUNKS[index]);
|
|
212
|
+
assert.equal(chunk.text, STREAM_CHUNKS[index].text);
|
|
213
|
+
});
|
|
214
|
+
// The wrapper still ran: cache tokens from the final chunk landed on the span.
|
|
215
|
+
assert.equal(span.attributes['gen_ai.usage.cache_read_input_tokens'], 32);
|
|
216
|
+
} finally {
|
|
217
|
+
teardown();
|
|
218
|
+
}
|
|
219
|
+
});
|