weflayr 0.20.6 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +1 -1
- package/package.json +2 -2
- package/src/auto-instrument.js +64 -31
- package/src/instrumentation/anthropic/get_cache_and_reasoning_telemetry.js +107 -0
- package/src/instrumentation/anthropic/get_streaming_stop_reason_telemetry.js +121 -0
- package/src/instrumentation/anthropic/get_tool_definitions_telemetry.js +50 -0
- package/src/instrumentation/anthropic/set_telemetry_attributes.js +37 -0
- package/src/instrumentation/bedrock/get_cache_telemetry.js +104 -0
- package/src/instrumentation/bedrock/get_input_message_content_telemetry.js +94 -0
- package/src/instrumentation/bedrock/get_streaming_output_message_telemetry.js +87 -0
- package/src/instrumentation/bedrock/set_telemetry_attributes.js +36 -0
- package/src/instrumentation/cohere/get_chat_telemetry.js +88 -0
- package/src/instrumentation/cohere/get_embeddings_telemetry.js +107 -0
- package/src/instrumentation/cohere/set_telemetry_attributes.js +32 -0
- package/src/instrumentation/google/get_audio_telemetry.js +30 -0
- package/src/instrumentation/google/get_cache_telemetry.js +25 -0
- package/src/instrumentation/google/get_embeddings_telemetry.js +99 -0
- package/src/instrumentation/google/get_provider_name.js +25 -0
- package/src/instrumentation/google/get_reasoning_telemetry.js +21 -0
- package/src/instrumentation/google/set_telemetry_attributes.js +98 -0
- package/src/instrumentation/openai/get_audio_telemetry.js +149 -0
- package/src/instrumentation/openai/get_cache_and_reasoning_telemetry.js +71 -0
- package/src/instrumentation/openai/get_embeddings_telemetry.js +107 -0
- package/src/instrumentation/openai/get_streaming_usage.js +81 -0
- package/src/instrumentation/openai/set_telemetry_attributes.js +39 -0
- package/src/otel/exporter.js +1 -6
- package/src/otel/propagation.js +5 -0
- package/src/otel/span-filter.js +0 -5
- package/src/otel/span-processor.js +30 -1
- package/src/setup.js +1 -2
- package/tests/index.test.js +144 -44
- package/tests/instrumentation/anthropic/get_cache_and_reasoning_telemetry.test.js +158 -0
- package/tests/instrumentation/anthropic/get_streaming_stop_reason_telemetry.test.js +117 -0
- package/tests/instrumentation/anthropic/get_tool_definitions_telemetry.test.js +82 -0
- package/tests/instrumentation/bedrock/get_cache_telemetry.test.js +102 -0
- package/tests/instrumentation/bedrock/get_input_message_content_telemetry.test.js +89 -0
- package/tests/instrumentation/bedrock/get_streaming_output_message_telemetry.test.js +87 -0
- package/tests/instrumentation/cohere/get_chat_telemetry.test.js +148 -0
- package/tests/instrumentation/cohere/get_embeddings_telemetry.test.js +132 -0
- package/tests/instrumentation/{elevenlabs.test.js → elevenlabs/full_instrumentation.test.js} +21 -5
- package/tests/instrumentation/google/get_embeddings_telemetry.test.js +128 -0
- package/tests/instrumentation/google/get_reasoning_telemetry.test.js +41 -0
- package/tests/instrumentation/google/set_telemetry_attributes.test.js +219 -0
- package/tests/instrumentation/openai/get_audio_telemetry.test.js +170 -0
- package/tests/instrumentation/openai/get_cache_and_reasoning_telemetry.test.js +83 -0
- package/tests/instrumentation/openai/get_embeddings_telemetry.test.js +145 -0
- package/tests/instrumentation/openai/get_streaming_usage.test.js +105 -0
- package/src/otel/openai-streaming-usage.js +0 -60
- /package/src/instrumentation/{elevenlabs.js → elevenlabs/full_instrumentation.js} +0 -0
package/tests/index.test.js
CHANGED
|
@@ -14,7 +14,6 @@ const { _state } = require('../src/setup');
|
|
|
14
14
|
const { isDefaultExportSpan } = require('../src/otel/span-filter');
|
|
15
15
|
const { WeflayrSpanProcessor } = require('../src/otel/span-processor');
|
|
16
16
|
const { buildExporter, _exportWithRetry } = require('../src/otel/exporter');
|
|
17
|
-
const { patchOpenAIStreamingUsage } = require('../src/otel/openai-streaming-usage');
|
|
18
17
|
|
|
19
18
|
const contextManager = new AsyncHooksContextManager();
|
|
20
19
|
contextManager.enable();
|
|
@@ -185,6 +184,54 @@ test('buildExporter strips trailing slash on baseUrl', () => {
|
|
|
185
184
|
assert.equal(_exporterParams(exporter).url, 'https://api.weflayr.com/');
|
|
186
185
|
});
|
|
187
186
|
|
|
187
|
+
test('buildExporter warns on every export error when warnOnExportError is true', async () => {
|
|
188
|
+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
|
|
189
|
+
const origExport = OTLPTraceExporter.prototype.export;
|
|
190
|
+
OTLPTraceExporter.prototype.export = function (spans, cb) {
|
|
191
|
+
cb({ code: 1, error: { code: 401, message: 'Unauthorized' } });
|
|
192
|
+
};
|
|
193
|
+
const warnings = [];
|
|
194
|
+
const origWarn = console.warn;
|
|
195
|
+
console.warn = (...args) => warnings.push(args.join(' '));
|
|
196
|
+
try {
|
|
197
|
+
const exporter = buildExporter({
|
|
198
|
+
baseUrl: 'https://api.weflayr.com',
|
|
199
|
+
apiKey: 'bad',
|
|
200
|
+
warnOnExportError: true,
|
|
201
|
+
});
|
|
202
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
203
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
204
|
+
assert.equal(warnings.filter((w) => w.includes('[weflayr]')).length, 2);
|
|
205
|
+
} finally {
|
|
206
|
+
OTLPTraceExporter.prototype.export = origExport;
|
|
207
|
+
console.warn = origWarn;
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test('buildExporter does not warn when warnOnExportError is false', async () => {
|
|
212
|
+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
|
|
213
|
+
const origExport = OTLPTraceExporter.prototype.export;
|
|
214
|
+
OTLPTraceExporter.prototype.export = function (spans, cb) {
|
|
215
|
+
cb({ code: 1, error: { code: 401, message: 'Unauthorized' } });
|
|
216
|
+
};
|
|
217
|
+
const warnings = [];
|
|
218
|
+
const origWarn = console.warn;
|
|
219
|
+
console.warn = (...args) => warnings.push(args.join(' '));
|
|
220
|
+
try {
|
|
221
|
+
const exporter = buildExporter({
|
|
222
|
+
baseUrl: 'https://api.weflayr.com',
|
|
223
|
+
apiKey: 'bad',
|
|
224
|
+
warnOnExportError: false,
|
|
225
|
+
});
|
|
226
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
227
|
+
await new Promise((resolve) => exporter.export([], resolve));
|
|
228
|
+
assert.equal(warnings.filter((w) => w.includes('[weflayr]')).length, 0);
|
|
229
|
+
} finally {
|
|
230
|
+
OTLPTraceExporter.prototype.export = origExport;
|
|
231
|
+
console.warn = origWarn;
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
|
|
188
235
|
// ── span-filter ───────────────────────────────────────────────────────────────
|
|
189
236
|
|
|
190
237
|
test('isDefaultExportSpan accepts spans with gen_ai.* attributes', () => {
|
|
@@ -288,6 +335,60 @@ test('WeflayrSpanProcessor applies mask before export', async () => {
|
|
|
288
335
|
assert.equal(span.attributes['gen_ai.prompt'], '[REDACTED]');
|
|
289
336
|
});
|
|
290
337
|
|
|
338
|
+
test('WeflayrSpanProcessor strips message content when capture disabled', async () => {
|
|
339
|
+
// Instrumentors gate content on their own env vars (openllmetry captures by
|
|
340
|
+
// default), so the processor itself must drop message content when
|
|
341
|
+
// captureMessageContent=false recorded "false" in the env var.
|
|
342
|
+
process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = 'false';
|
|
343
|
+
try {
|
|
344
|
+
const exporter = makeExporterStub();
|
|
345
|
+
const proc = new WeflayrSpanProcessor({ exporter, exportMode: 'immediate' });
|
|
346
|
+
const span = makeFakeSpan({
|
|
347
|
+
attributes: {
|
|
348
|
+
'gen_ai.system': 'openai',
|
|
349
|
+
'gen_ai.input.messages': '[{"role": "user"}]',
|
|
350
|
+
'gen_ai.output.messages': '[{"role": "assistant"}]',
|
|
351
|
+
'gen_ai.system_instructions': '[{"type": "text"}]',
|
|
352
|
+
'gen_ai.prompt.0.content': 'Hi Noe',
|
|
353
|
+
'gen_ai.completion.0.content': 'Hello!',
|
|
354
|
+
'gen_ai.usage.input_tokens': 12,
|
|
355
|
+
},
|
|
356
|
+
});
|
|
357
|
+
proc.onEnd(span);
|
|
358
|
+
await proc.forceFlush();
|
|
359
|
+
assert.equal(span.attributes['gen_ai.system'], 'openai');
|
|
360
|
+
assert.equal(span.attributes['gen_ai.usage.input_tokens'], 12);
|
|
361
|
+
for (const key of Object.keys(span.attributes)) {
|
|
362
|
+
assert.ok(!key.startsWith('gen_ai.prompt'), `unexpected ${key}`);
|
|
363
|
+
assert.ok(!key.startsWith('gen_ai.completion'), `unexpected ${key}`);
|
|
364
|
+
}
|
|
365
|
+
assert.ok(!('gen_ai.input.messages' in span.attributes));
|
|
366
|
+
assert.ok(!('gen_ai.output.messages' in span.attributes));
|
|
367
|
+
assert.ok(!('gen_ai.system_instructions' in span.attributes));
|
|
368
|
+
} finally {
|
|
369
|
+
delete process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT;
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
test('WeflayrSpanProcessor keeps message content when capture enabled', async () => {
|
|
374
|
+
process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT = 'true';
|
|
375
|
+
try {
|
|
376
|
+
const exporter = makeExporterStub();
|
|
377
|
+
const proc = new WeflayrSpanProcessor({ exporter, exportMode: 'immediate' });
|
|
378
|
+
const span = makeFakeSpan({
|
|
379
|
+
attributes: {
|
|
380
|
+
'gen_ai.system': 'openai',
|
|
381
|
+
'gen_ai.input.messages': '[{"role": "user"}]',
|
|
382
|
+
},
|
|
383
|
+
});
|
|
384
|
+
proc.onEnd(span);
|
|
385
|
+
await proc.forceFlush();
|
|
386
|
+
assert.equal(span.attributes['gen_ai.input.messages'], '[{"role": "user"}]');
|
|
387
|
+
} finally {
|
|
388
|
+
delete process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT;
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
|
|
291
392
|
// ── propagateMetadata ────────────────────────────────────────────────────────
|
|
292
393
|
|
|
293
394
|
function _makeMetadataHarness() {
|
|
@@ -478,12 +579,48 @@ test('propagateMetadata providerName writes attribute', async () => {
|
|
|
478
579
|
await provider.shutdown();
|
|
479
580
|
});
|
|
480
581
|
|
|
582
|
+
test('propagateMetadata requires featureName and customerId', () => {
|
|
583
|
+
assert.throws(() => weflayr.propagateMetadata({}), /featureName.*customerId.*required/);
|
|
584
|
+
assert.throws(() => weflayr.propagateMetadata({ featureName: 'f' }), /required/);
|
|
585
|
+
assert.throws(() => weflayr.propagateMetadata({ customerId: 'c' }), /required/);
|
|
586
|
+
});
|
|
587
|
+
|
|
481
588
|
// ── autoInstrument ────────────────────────────────────────────────────────────
|
|
482
589
|
|
|
483
590
|
function _ensureSetupDone() {
|
|
484
591
|
if (!_state.processor) weflayr.setup({ apiKey: 'test-key' });
|
|
485
592
|
}
|
|
486
593
|
|
|
594
|
+
// Instrumenting OPENAI also runs the bundled audio + embeddings instrumentors,
|
|
595
|
+
// which eagerly require the openai resource modules. Stub them so autoInstrument
|
|
596
|
+
// tests don't need the real `openai` dependency installed.
|
|
597
|
+
const OPENAI_RESOURCE_STUBS = {
|
|
598
|
+
'openai/resources/audio/speech': {
|
|
599
|
+
Speech: class {
|
|
600
|
+
create() {}
|
|
601
|
+
},
|
|
602
|
+
},
|
|
603
|
+
'openai/resources/audio/transcriptions': {
|
|
604
|
+
Transcriptions: class {
|
|
605
|
+
create() {}
|
|
606
|
+
},
|
|
607
|
+
},
|
|
608
|
+
'openai/resources/embeddings': {
|
|
609
|
+
Embeddings: class {
|
|
610
|
+
create() {}
|
|
611
|
+
},
|
|
612
|
+
},
|
|
613
|
+
};
|
|
614
|
+
|
|
615
|
+
function _stubOpenAIResources() {
|
|
616
|
+
for (const [id, exports] of Object.entries(OPENAI_RESOURCE_STUBS)) {
|
|
617
|
+
require.cache[id] = { id, filename: id, loaded: true, exports };
|
|
618
|
+
}
|
|
619
|
+
return () => {
|
|
620
|
+
for (const id of Object.keys(OPENAI_RESOURCE_STUBS)) delete require.cache[id];
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
|
|
487
624
|
test('autoInstrument throws "install package X" when the dep is missing', async () => {
|
|
488
625
|
_ensureSetupDone();
|
|
489
626
|
await assert.rejects(
|
|
@@ -517,7 +654,7 @@ test('autoInstrument loads a stubbed Traceloop package', async () => {
|
|
|
517
654
|
}
|
|
518
655
|
const origResolve = Module._resolveFilename;
|
|
519
656
|
Module._resolveFilename = function patched(req, ...rest) {
|
|
520
|
-
if (req === '@traceloop/instrumentation-openai') return req;
|
|
657
|
+
if (req === '@traceloop/instrumentation-openai' || req in OPENAI_RESOURCE_STUBS) return req;
|
|
521
658
|
return origResolve.call(this, req, ...rest);
|
|
522
659
|
};
|
|
523
660
|
require.cache['@traceloop/instrumentation-openai'] = {
|
|
@@ -526,12 +663,14 @@ test('autoInstrument loads a stubbed Traceloop package', async () => {
|
|
|
526
663
|
loaded: true,
|
|
527
664
|
exports: { OpenAIInstrumentation: FakeOpenAIInstrumentation },
|
|
528
665
|
};
|
|
666
|
+
const cleanupOpenAI = _stubOpenAIResources();
|
|
529
667
|
try {
|
|
530
668
|
await weflayr.autoInstrument({ aiSdks: weflayr.AiSdk.OPENAI });
|
|
531
669
|
assert.equal(constructed, true);
|
|
532
670
|
} finally {
|
|
533
671
|
Module._resolveFilename = origResolve;
|
|
534
672
|
delete require.cache['@traceloop/instrumentation-openai'];
|
|
673
|
+
cleanupOpenAI();
|
|
535
674
|
}
|
|
536
675
|
});
|
|
537
676
|
|
|
@@ -564,7 +703,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
|
|
|
564
703
|
}
|
|
565
704
|
const origResolve = Module._resolveFilename;
|
|
566
705
|
Module._resolveFilename = function patched(req, ...rest) {
|
|
567
|
-
if (req === '@traceloop/instrumentation-openai') return req;
|
|
706
|
+
if (req === '@traceloop/instrumentation-openai' || req in OPENAI_RESOURCE_STUBS) return req;
|
|
568
707
|
return origResolve.call(this, req, ...rest);
|
|
569
708
|
};
|
|
570
709
|
require.cache['@traceloop/instrumentation-openai'] = {
|
|
@@ -573,6 +712,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
|
|
|
573
712
|
loaded: true,
|
|
574
713
|
exports: { OpenAIInstrumentation: FakeOpenAIInstrumentation },
|
|
575
714
|
};
|
|
715
|
+
const cleanupOpenAI = _stubOpenAIResources();
|
|
576
716
|
try {
|
|
577
717
|
await weflayr.autoInstrument({
|
|
578
718
|
aiSdks: weflayr.AiSdk.OPENAI,
|
|
@@ -582,6 +722,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
|
|
|
582
722
|
} finally {
|
|
583
723
|
Module._resolveFilename = origResolve;
|
|
584
724
|
delete require.cache['@traceloop/instrumentation-openai'];
|
|
725
|
+
cleanupOpenAI();
|
|
585
726
|
}
|
|
586
727
|
});
|
|
587
728
|
|
|
@@ -641,47 +782,6 @@ test('exporter gives up after maxRetries on persistent 503', async () => {
|
|
|
641
782
|
assert.equal(calls, 3);
|
|
642
783
|
});
|
|
643
784
|
|
|
644
|
-
// ── OpenAI streaming usage backfill ─────────────────────────────────────────────
|
|
645
|
-
|
|
646
|
-
// Build a fake OpenAI instrumentor whose streaming handler mimics traceloop's:
|
|
647
|
-
// it yields content chunks, then a final usage chunk (as OpenAI sends when
|
|
648
|
-
// `stream_options.include_usage` is set) without recording usage itself. The
|
|
649
|
-
// patch should read the usage chunk and set the token attributes on the span.
|
|
650
|
-
function makeFakeOpenAIInstrumentation() {
|
|
651
|
-
return {
|
|
652
|
-
_streamingWrapPromise() {
|
|
653
|
-
return (async function* originalStream() {
|
|
654
|
-
yield { id: 'chatcmpl-1', model: 'gpt-4o-mini', choices: [{ delta: { content: 'Hi' } }] };
|
|
655
|
-
yield {
|
|
656
|
-
id: 'chatcmpl-1',
|
|
657
|
-
model: 'gpt-4o-mini',
|
|
658
|
-
choices: [],
|
|
659
|
-
usage: { prompt_tokens: 13, completion_tokens: 2, total_tokens: 15 },
|
|
660
|
-
};
|
|
661
|
-
})();
|
|
662
|
-
},
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
test('patchOpenAIStreamingUsage sets token attributes from the streamed usage chunk', async () => {
|
|
667
|
-
const span = makeFakeSpan();
|
|
668
|
-
const instrumentation = makeFakeOpenAIInstrumentation();
|
|
669
|
-
patchOpenAIStreamingUsage(instrumentation);
|
|
670
|
-
|
|
671
|
-
const stream = instrumentation._streamingWrapPromise({ span });
|
|
672
|
-
for await (const chunk of stream) void chunk;
|
|
673
|
-
|
|
674
|
-
assert.equal(span.attributes['gen_ai.usage.input_tokens'], 13);
|
|
675
|
-
assert.equal(span.attributes['gen_ai.usage.output_tokens'], 2);
|
|
676
|
-
assert.equal(span.attributes['gen_ai.usage.total_tokens'], 15);
|
|
677
|
-
});
|
|
678
|
-
|
|
679
|
-
test('patchOpenAIStreamingUsage is a no-op when the streaming handler is absent', () => {
|
|
680
|
-
const instrumentation = {};
|
|
681
|
-
patchOpenAIStreamingUsage(instrumentation);
|
|
682
|
-
assert.equal(instrumentation._streamingWrapPromise, undefined);
|
|
683
|
-
});
|
|
684
|
-
|
|
685
785
|
// ── flush ─────────────────────────────────────────────────────────────────────
|
|
686
786
|
|
|
687
787
|
test('flush does not throw', async () => {
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Anthropic cached-prompt token complement. It wraps the primary
|
|
4
|
+
// instrumentor's `_endSpan({ span, type, result })` to stamp
|
|
5
|
+
// gen_ai.usage.cache_read_input_tokens / cache_creation_input_tokens from the
|
|
6
|
+
// response's `usage.cache_read_input_tokens` / `cache_creation_input_tokens`.
|
|
7
|
+
|
|
8
|
+
const test = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
|
|
11
|
+
const {
|
|
12
|
+
AnthropicPromptCachingInstrumentor,
|
|
13
|
+
} = require('../../../src/instrumentation/anthropic/get_cache_and_reasoning_telemetry');
|
|
14
|
+
|
|
15
|
+
function fakeSpan() {
|
|
16
|
+
const attrs = {};
|
|
17
|
+
return {
|
|
18
|
+
attrs,
|
|
19
|
+
isRecording: () => true,
|
|
20
|
+
setAttribute(key, value) {
|
|
21
|
+
attrs[key] = value;
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
test('stamps cache write and still calls the original _endSpan', () => {
|
|
27
|
+
const calls = [];
|
|
28
|
+
const instrumentation = {
|
|
29
|
+
_endSpan(args) {
|
|
30
|
+
calls.push(args);
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
34
|
+
|
|
35
|
+
const span = fakeSpan();
|
|
36
|
+
instrumentation._endSpan({
|
|
37
|
+
span,
|
|
38
|
+
type: 'completion',
|
|
39
|
+
result: {
|
|
40
|
+
usage: { input_tokens: 10, cache_creation_input_tokens: 5269, cache_read_input_tokens: 0 },
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], 5269);
|
|
45
|
+
assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
|
|
46
|
+
assert.equal(calls.length, 1);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('stamps cache read on a warm call', () => {
|
|
50
|
+
const instrumentation = { _endSpan() {} };
|
|
51
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
52
|
+
|
|
53
|
+
const span = fakeSpan();
|
|
54
|
+
instrumentation._endSpan({
|
|
55
|
+
span,
|
|
56
|
+
type: 'completion',
|
|
57
|
+
result: {
|
|
58
|
+
usage: { input_tokens: 10, cache_creation_input_tokens: 0, cache_read_input_tokens: 5271 },
|
|
59
|
+
},
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 5271);
|
|
63
|
+
assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test('sets nothing for an uncached call', () => {
|
|
67
|
+
const instrumentation = { _endSpan() {} };
|
|
68
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
69
|
+
|
|
70
|
+
const span = fakeSpan();
|
|
71
|
+
instrumentation._endSpan({ span, type: 'completion', result: { usage: { input_tokens: 13 } } });
|
|
72
|
+
|
|
73
|
+
assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
|
|
74
|
+
assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('stamps extended thinking tokens from output_tokens_details', () => {
|
|
78
|
+
const instrumentation = { _endSpan() {} };
|
|
79
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
80
|
+
|
|
81
|
+
const span = fakeSpan();
|
|
82
|
+
instrumentation._endSpan({
|
|
83
|
+
span,
|
|
84
|
+
type: 'completion',
|
|
85
|
+
result: { usage: { input_tokens: 10, output_tokens_details: { thinking_tokens: 150 } } },
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
assert.equal(span.attrs['gen_ai.usage.output_tokens_details.thinking_tokens'], 150);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('stamps cache + raw input from the streamed message_start usage', async () => {
|
|
92
|
+
const instrumentation = {
|
|
93
|
+
_endSpan() {},
|
|
94
|
+
_streamingWrapPromise() {
|
|
95
|
+
return (async function* () {
|
|
96
|
+
yield {
|
|
97
|
+
type: 'message_start',
|
|
98
|
+
message: {
|
|
99
|
+
usage: {
|
|
100
|
+
input_tokens: 10,
|
|
101
|
+
cache_read_input_tokens: 5270,
|
|
102
|
+
cache_creation_input_tokens: 0,
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
yield { type: 'content_block_delta', delta: { text: 'OK' } };
|
|
107
|
+
})();
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
111
|
+
|
|
112
|
+
const span = fakeSpan();
|
|
113
|
+
for await (const _chunk of instrumentation._streamingWrapPromise('client', 'mod', { span })) {
|
|
114
|
+
void _chunk;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 5270);
|
|
118
|
+
assert.equal(span.attrs['gen_ai.usage.input_tokens'], 10);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('re-yields every stream chunk unchanged — no telemetry collision with the payload', async () => {
|
|
122
|
+
const emitted = [
|
|
123
|
+
{
|
|
124
|
+
type: 'message_start',
|
|
125
|
+
message: { usage: { input_tokens: 10, cache_read_input_tokens: 5270 } },
|
|
126
|
+
},
|
|
127
|
+
{ type: 'content_block_delta', delta: { text: 'O' } },
|
|
128
|
+
{ type: 'content_block_delta', delta: { text: 'K' } },
|
|
129
|
+
];
|
|
130
|
+
const instrumentation = {
|
|
131
|
+
_endSpan() {},
|
|
132
|
+
_streamingWrapPromise() {
|
|
133
|
+
return (async function* () {
|
|
134
|
+
for (const chunk of emitted) yield chunk;
|
|
135
|
+
})();
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
139
|
+
|
|
140
|
+
const span = fakeSpan();
|
|
141
|
+
const collected = [];
|
|
142
|
+
for await (const chunk of instrumentation._streamingWrapPromise('client', 'mod', { span })) {
|
|
143
|
+
collected.push(chunk);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Same count, order, and object identity: observeCache re-yields each chunk by
|
|
147
|
+
// reference without copying or mutating it.
|
|
148
|
+
assert.equal(collected.length, emitted.length);
|
|
149
|
+
collected.forEach((chunk, index) => assert.equal(chunk, emitted[index]));
|
|
150
|
+
// …while still stamping cache + raw input off the message_start chunk.
|
|
151
|
+
assert.equal(span.attrs['gen_ai.usage.input_tokens'], 10);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test('is a no-op when the instrumentor has no _endSpan', () => {
|
|
155
|
+
const instrumentation = {};
|
|
156
|
+
new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
|
|
157
|
+
assert.equal(instrumentation._endSpan, undefined);
|
|
158
|
+
});
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Anthropic streamed stop-reason complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed the way the real one works:
|
|
6
|
+
// `_streamingWrapPromise` returns `new moduleExports.APIPromise(...)` whose
|
|
7
|
+
// parse step wraps the SDK stream with an accumulating iterator that calls
|
|
8
|
+
// `_endSpan` when exhausted — so the test exercises the complement's APIPromise
|
|
9
|
+
// interposition end to end.
|
|
10
|
+
|
|
11
|
+
const test = require('node:test');
|
|
12
|
+
const assert = require('node:assert/strict');
|
|
13
|
+
|
|
14
|
+
const {
|
|
15
|
+
AnthropicStreamingStopReasonInstrumentor,
|
|
16
|
+
} = require('../../../src/instrumentation/anthropic/get_streaming_stop_reason_telemetry');
|
|
17
|
+
|
|
18
|
+
class StubStream {
|
|
19
|
+
constructor(iteratorFactory, controller) {
|
|
20
|
+
this._iteratorFactory = iteratorFactory;
|
|
21
|
+
this.controller = controller;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
[Symbol.asyncIterator]() {
|
|
25
|
+
return this._iteratorFactory();
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
class StubAPIPromise {
|
|
30
|
+
constructor(client, responsePromise, parseResponse) {
|
|
31
|
+
this.parsed = parseResponse(client, responsePromise);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function makeStubInstrumentation() {
|
|
36
|
+
const finalized = [];
|
|
37
|
+
return {
|
|
38
|
+
finalized,
|
|
39
|
+
_streamingWrapPromise(client, moduleExports, { span, promise }) {
|
|
40
|
+
const self = this;
|
|
41
|
+
return new moduleExports.APIPromise(
|
|
42
|
+
client,
|
|
43
|
+
promise,
|
|
44
|
+
async (parseClient, realStreamPromise) => {
|
|
45
|
+
const realStream = await realStreamPromise;
|
|
46
|
+
// Mimics traceloop: re-wrap the stream with an accumulating iterator
|
|
47
|
+
// that finalizes the span once exhausted.
|
|
48
|
+
return new realStream.constructor(
|
|
49
|
+
() =>
|
|
50
|
+
(async function* accumulate() {
|
|
51
|
+
const result = { stop_reason: null };
|
|
52
|
+
for await (const chunk of realStream) yield chunk;
|
|
53
|
+
self._endSpan({ span, result });
|
|
54
|
+
})(),
|
|
55
|
+
realStream.controller
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
);
|
|
59
|
+
},
|
|
60
|
+
_endSpan(args) {
|
|
61
|
+
finalized.push(args.result);
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function chunkStream() {
|
|
67
|
+
return new StubStream(async function* chunks() {
|
|
68
|
+
yield { type: 'message_start', message: { usage: { input_tokens: 5 } } };
|
|
69
|
+
yield { type: 'content_block_delta', delta: { type: 'text_delta', text: 'Hello' } };
|
|
70
|
+
yield {
|
|
71
|
+
type: 'message_delta',
|
|
72
|
+
delta: { stop_reason: 'end_turn' },
|
|
73
|
+
usage: { output_tokens: 3 },
|
|
74
|
+
};
|
|
75
|
+
yield { type: 'message_stop' };
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
test('injects the streamed stop reason into the result before endSpan', async () => {
|
|
80
|
+
const instrumentation = makeStubInstrumentation();
|
|
81
|
+
new AnthropicStreamingStopReasonInstrumentor().instrument(instrumentation);
|
|
82
|
+
|
|
83
|
+
const span = { name: 'anthropic.chat' };
|
|
84
|
+
const apiPromise = instrumentation._streamingWrapPromise(
|
|
85
|
+
{},
|
|
86
|
+
{ APIPromise: StubAPIPromise },
|
|
87
|
+
{ span, type: 'chat', promise: Promise.resolve(chunkStream()) }
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
const stream = await apiPromise.parsed;
|
|
91
|
+
const seen = [];
|
|
92
|
+
for await (const chunk of stream) seen.push(chunk.type);
|
|
93
|
+
|
|
94
|
+
assert.deepEqual(seen, ['message_start', 'content_block_delta', 'message_delta', 'message_stop']);
|
|
95
|
+
assert.equal(instrumentation.finalized.length, 1);
|
|
96
|
+
assert.equal(instrumentation.finalized[0].stop_reason, 'end_turn');
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('leaves an already-set stop reason untouched', () => {
|
|
100
|
+
const instrumentation = makeStubInstrumentation();
|
|
101
|
+
new AnthropicStreamingStopReasonInstrumentor().instrument(instrumentation);
|
|
102
|
+
|
|
103
|
+
const span = { name: 'anthropic.chat' };
|
|
104
|
+
instrumentation._endSpan({ span, result: { stop_reason: 'max_tokens' } });
|
|
105
|
+
assert.equal(instrumentation.finalized[0].stop_reason, 'max_tokens');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test('uninstrument restores the original methods', () => {
|
|
109
|
+
const instrumentation = makeStubInstrumentation();
|
|
110
|
+
const streamOriginal = instrumentation._streamingWrapPromise;
|
|
111
|
+
const endSpanOriginal = instrumentation._endSpan;
|
|
112
|
+
const instrumentor = new AnthropicStreamingStopReasonInstrumentor();
|
|
113
|
+
instrumentor.instrument(instrumentation);
|
|
114
|
+
instrumentor.uninstrument();
|
|
115
|
+
assert.equal(instrumentation._streamingWrapPromise, streamOriginal);
|
|
116
|
+
assert.equal(instrumentation._endSpan, endSpanOriginal);
|
|
117
|
+
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tests for the Anthropic tool-definitions complement.
|
|
4
|
+
//
|
|
5
|
+
// The traceloop instrumentation instance is stubbed with a `startSpan` that
|
|
6
|
+
// returns a fake span, so the wrap can be exercised without the real SDK.
|
|
7
|
+
|
|
8
|
+
const test = require('node:test');
|
|
9
|
+
const assert = require('node:assert/strict');
|
|
10
|
+
|
|
11
|
+
const {
|
|
12
|
+
AnthropicToolDefinitionsInstrumentor,
|
|
13
|
+
} = require('../../../src/instrumentation/anthropic/get_tool_definitions_telemetry');
|
|
14
|
+
|
|
15
|
+
function makeFakeSpan() {
|
|
16
|
+
const attributes = {};
|
|
17
|
+
return {
|
|
18
|
+
attributes,
|
|
19
|
+
setAttribute(key, value) {
|
|
20
|
+
attributes[key] = value;
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function makeStubInstrumentation() {
|
|
26
|
+
return {
|
|
27
|
+
startSpan(request) {
|
|
28
|
+
const span = makeFakeSpan();
|
|
29
|
+
span.attributes['gen_ai.request.model'] = request.params.model;
|
|
30
|
+
return span;
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const WEATHER_TOOL = {
|
|
36
|
+
name: 'get_weather',
|
|
37
|
+
description: 'Get the current weather for a city.',
|
|
38
|
+
input_schema: { type: 'object', properties: { city: { type: 'string' } } },
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
test('stamps tool definitions from the request params', () => {
|
|
42
|
+
const instrumentation = makeStubInstrumentation();
|
|
43
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
44
|
+
instrumentor.instrument(instrumentation);
|
|
45
|
+
|
|
46
|
+
const span = instrumentation.startSpan({
|
|
47
|
+
type: 'chat',
|
|
48
|
+
params: { model: 'claude-haiku-4-5', tools: [WEATHER_TOOL] },
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
assert.equal(span.attributes['gen_ai.request.model'], 'claude-haiku-4-5');
|
|
52
|
+
assert.deepEqual(JSON.parse(span.attributes['gen_ai.tool.definitions']), [WEATHER_TOOL]);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('no attribute when the request has no tools', () => {
|
|
56
|
+
const instrumentation = makeStubInstrumentation();
|
|
57
|
+
new AnthropicToolDefinitionsInstrumentor().instrument(instrumentation);
|
|
58
|
+
|
|
59
|
+
const span = instrumentation.startSpan({
|
|
60
|
+
type: 'chat',
|
|
61
|
+
params: { model: 'claude-haiku-4-5' },
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
assert.ok(!('gen_ai.tool.definitions' in span.attributes));
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test('uninstrument restores the original startSpan', () => {
|
|
68
|
+
const instrumentation = makeStubInstrumentation();
|
|
69
|
+
const original = instrumentation.startSpan;
|
|
70
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
71
|
+
instrumentor.instrument(instrumentation);
|
|
72
|
+
assert.notEqual(instrumentation.startSpan, original);
|
|
73
|
+
instrumentor.uninstrument();
|
|
74
|
+
assert.equal(instrumentation.startSpan, original);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test('is a no-op when the instrumentor has no startSpan', () => {
|
|
78
|
+
const instrumentor = new AnthropicToolDefinitionsInstrumentor();
|
|
79
|
+
instrumentor.instrument({});
|
|
80
|
+
instrumentor.instrument(undefined);
|
|
81
|
+
instrumentor.uninstrument();
|
|
82
|
+
});
|