weflayr 0.29.1 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +9 -9
- package/dist/src/api/generated/facade.gen.d.ts +1436 -10
- package/dist/src/api/generated/facade.gen.js +24 -1
- package/dist/src/api/generated/index.d.ts +2 -2
- package/dist/src/api/generated/index.js +24 -1
- package/dist/src/api/generated/sdk.gen.d.ts +97 -5
- package/dist/src/api/generated/sdk.gen.js +256 -5
- package/dist/src/api/generated/types.gen.d.ts +4527 -741
- package/dist/src/get-data-uri-mime-type.d.ts +8 -0
- package/dist/src/get-data-uri-mime-type.js +18 -0
- package/dist/src/instrumentation/anthropic/get_streaming_accumulation_telemetry.d.ts +6 -4
- package/dist/src/instrumentation/anthropic/get_streaming_accumulation_telemetry.js +22 -5
- package/dist/src/instrumentation/bedrock/{get_cache_telemetry.d.ts → get_cache_and_reasoning_telemetry.d.ts} +7 -0
- package/dist/src/instrumentation/bedrock/{get_cache_telemetry.js → get_cache_and_reasoning_telemetry.js} +25 -2
- package/dist/src/instrumentation/bedrock/get_input_message_content_telemetry.js +23 -16
- package/dist/src/instrumentation/bedrock/get_streaming_output_message_telemetry.d.ts +6 -0
- package/dist/src/instrumentation/bedrock/get_streaming_output_message_telemetry.js +18 -2
- package/dist/src/instrumentation/bedrock/set_telemetry_attributes.js +2 -2
- package/dist/src/instrumentation/cohere/get_input_message_content_telemetry.d.ts +2 -3
- package/dist/src/instrumentation/cohere/get_input_message_content_telemetry.js +6 -10
- package/dist/src/instrumentation/google/get_cache_operations_telemetry.js +7 -3
- package/dist/src/instrumentation/mistral/full_instrumentation.d.ts +10 -4
- package/dist/src/instrumentation/mistral/full_instrumentation.js +47 -14
- package/dist/src/instrumentation/mistral/get_input_message_content_telemetry.d.ts +1 -1
- package/dist/src/instrumentation/mistral/get_input_message_content_telemetry.js +5 -8
- package/dist/src/instrumentation/mistral/get_streamed_reasoning_telemetry.d.ts +38 -0
- package/dist/src/instrumentation/mistral/get_streamed_reasoning_telemetry.js +72 -0
- package/dist/src/instrumentation/openai/get_file_part_mime_type_telemetry.d.ts +5 -4
- package/dist/src/instrumentation/openai/get_file_part_mime_type_telemetry.js +14 -10
- package/dist/src/instrumentation/openai/get_reasoning_effort_telemetry.d.ts +16 -0
- package/dist/src/instrumentation/openai/get_reasoning_effort_telemetry.js +49 -0
- package/dist/src/instrumentation/openai/get_responses_telemetry.d.ts +7 -7
- package/dist/src/instrumentation/openai/get_responses_telemetry.js +11 -11
- package/dist/src/instrumentation/openai/get_streaming_usage.d.ts +5 -2
- package/dist/src/instrumentation/openai/get_streaming_usage.js +10 -2
- package/dist/src/instrumentation/openai/set_telemetry_attributes.d.ts +2 -1
- package/dist/src/instrumentation/openai/set_telemetry_attributes.js +4 -1
- package/dist/src/otel/exporter.d.ts +1 -1
- package/dist/src/otel/span-filter.d.ts +1 -1
- package/dist/src/otel/span-processor.d.ts +1 -1
- package/dist/src/otel/span-processor.js +3 -7
- package/dist/src/propagate-metadata-to-gateway.js +38 -11
- package/dist/src/setup.js +19 -7
- package/package.json +9 -9
- package/dist/src/otel/strip-media-payloads.d.ts +0 -26
- package/dist/src/otel/strip-media-payloads.js +0 -135
|
@@ -15,3 +15,11 @@
|
|
|
15
15
|
* `"https://example.com/cat.png"` -> `null`.
|
|
16
16
|
*/
|
|
17
17
|
export declare function getDataUriMimeType(uri: string): string | null;
|
|
18
|
+
/**
|
|
19
|
+
* The `blob` part holding the bytes of a base64 `data:` URI, or null when the
|
|
20
|
+
* string is not one. The ingestion API moves the bytes to S3.
|
|
21
|
+
*
|
|
22
|
+
* e.g. `"data:image/png;base64,AAA"` ->
|
|
23
|
+
* `{ type: "blob", mime_type: "image/png", content: "AAA" }`.
|
|
24
|
+
*/
|
|
25
|
+
export declare function getDataUriBlobPart(uri: string): Record<string, string> | null;
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.getDataUriMimeType = getDataUriMimeType;
|
|
13
|
+
exports.getDataUriBlobPart = getDataUriBlobPart;
|
|
13
14
|
const DATA_URI_PREFIX = 'data:';
|
|
14
15
|
/**
|
|
15
16
|
* The mime type of a `data:<mime_type>[;base64],<payload>` URI, or null when
|
|
@@ -23,3 +24,20 @@ function getDataUriMimeType(uri) {
|
|
|
23
24
|
return null;
|
|
24
25
|
return uri.slice(DATA_URI_PREFIX.length).split(';')[0].split(',')[0];
|
|
25
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* The `blob` part holding the bytes of a base64 `data:` URI, or null when the
|
|
29
|
+
* string is not one. The ingestion API moves the bytes to S3.
|
|
30
|
+
*
|
|
31
|
+
* e.g. `"data:image/png;base64,AAA"` ->
|
|
32
|
+
* `{ type: "blob", mime_type: "image/png", content: "AAA" }`.
|
|
33
|
+
*/
|
|
34
|
+
function getDataUriBlobPart(uri) {
|
|
35
|
+
const mimeType = getDataUriMimeType(uri);
|
|
36
|
+
const separatorIndex = uri.indexOf(',');
|
|
37
|
+
if (mimeType === null ||
|
|
38
|
+
separatorIndex === -1 ||
|
|
39
|
+
!uri.slice(0, separatorIndex).endsWith(';base64')) {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
return { type: 'blob', mime_type: mimeType, content: uri.slice(separatorIndex + 1) };
|
|
43
|
+
}
|
|
@@ -2,20 +2,22 @@
|
|
|
2
2
|
* Streamed-accumulation gap filling for the Anthropic chat span.
|
|
3
3
|
*
|
|
4
4
|
* `@traceloop/instrumentation-anthropic` accumulates a streamed response into a
|
|
5
|
-
* result object but drops
|
|
5
|
+
* result object but drops three things on the way:
|
|
6
6
|
* - the `message_delta` chunk's `delta.stop_reason`, so streamed spans format
|
|
7
|
-
* their output message with an empty `finish_reason`,
|
|
7
|
+
* their output message with an empty `finish_reason`,
|
|
8
8
|
* - the `content_block_delta` chunks' `input_json_delta` fragments, so a
|
|
9
9
|
* streamed tool call's `tool_call` part carries no `arguments`. This covers
|
|
10
10
|
* both a caller's own `tool_use` block and the `server_tool_use` block of a
|
|
11
|
-
* provider-run web search, whose query streams the same way
|
|
11
|
+
* provider-run web search, whose query streams the same way, and
|
|
12
|
+
* - the `content_block_delta` chunks' `thinking_delta` text, so a streamed
|
|
13
|
+
* thinking block's `reasoning` part is empty.
|
|
12
14
|
*
|
|
13
15
|
* `_streamingWrapPromise(client, moduleExports, { span, type, promise })`
|
|
14
16
|
* returns `new moduleExports.APIPromise(...)` whose parse step wraps the real
|
|
15
17
|
* SDK stream with the instrumentor's accumulating iterator. This complement
|
|
16
18
|
* hands it a patched `moduleExports` whose APIPromise interposes on that parse
|
|
17
19
|
* step, observing the chunks (outside the accumulating iterator, so every chunk
|
|
18
|
-
* is seen before the iterator finishes) to capture
|
|
20
|
+
* is seen before the iterator finishes) to capture them, which are then
|
|
19
21
|
* injected into the accumulated result just before `_endSpan` formats the
|
|
20
22
|
* output message.
|
|
21
23
|
*/
|
|
@@ -3,20 +3,22 @@
|
|
|
3
3
|
* Streamed-accumulation gap filling for the Anthropic chat span.
|
|
4
4
|
*
|
|
5
5
|
* `@traceloop/instrumentation-anthropic` accumulates a streamed response into a
|
|
6
|
-
* result object but drops
|
|
6
|
+
* result object but drops three things on the way:
|
|
7
7
|
* - the `message_delta` chunk's `delta.stop_reason`, so streamed spans format
|
|
8
|
-
* their output message with an empty `finish_reason`,
|
|
8
|
+
* their output message with an empty `finish_reason`,
|
|
9
9
|
* - the `content_block_delta` chunks' `input_json_delta` fragments, so a
|
|
10
10
|
* streamed tool call's `tool_call` part carries no `arguments`. This covers
|
|
11
11
|
* both a caller's own `tool_use` block and the `server_tool_use` block of a
|
|
12
|
-
* provider-run web search, whose query streams the same way
|
|
12
|
+
* provider-run web search, whose query streams the same way, and
|
|
13
|
+
* - the `content_block_delta` chunks' `thinking_delta` text, so a streamed
|
|
14
|
+
* thinking block's `reasoning` part is empty.
|
|
13
15
|
*
|
|
14
16
|
* `_streamingWrapPromise(client, moduleExports, { span, type, promise })`
|
|
15
17
|
* returns `new moduleExports.APIPromise(...)` whose parse step wraps the real
|
|
16
18
|
* SDK stream with the instrumentor's accumulating iterator. This complement
|
|
17
19
|
* hands it a patched `moduleExports` whose APIPromise interposes on that parse
|
|
18
20
|
* step, observing the chunks (outside the accumulating iterator, so every chunk
|
|
19
|
-
* is seen before the iterator finishes) to capture
|
|
21
|
+
* is seen before the iterator finishes) to capture them, which are then
|
|
20
22
|
* injected into the accumulated result just before `_endSpan` formats the
|
|
21
23
|
* output message.
|
|
22
24
|
*/
|
|
@@ -36,7 +38,7 @@ class AnthropicStreamingAccumulationInstrumentor {
|
|
|
36
38
|
function gapsFor(span) {
|
|
37
39
|
let gaps = gapsBySpan.get(span);
|
|
38
40
|
if (!gaps) {
|
|
39
|
-
gaps = { toolArgumentsJsonByIndex: new Map() };
|
|
41
|
+
gaps = { toolArgumentsJsonByIndex: new Map(), thinkingByIndex: new Map() };
|
|
40
42
|
gapsBySpan.set(span, gaps);
|
|
41
43
|
}
|
|
42
44
|
return gaps;
|
|
@@ -58,6 +60,14 @@ class AnthropicStreamingAccumulationInstrumentor {
|
|
|
58
60
|
const argumentsByIndex = gapsFor(span).toolArgumentsJsonByIndex;
|
|
59
61
|
argumentsByIndex.set(chunk.index, (argumentsByIndex.get(chunk.index) ?? '') + chunk.delta.partial_json);
|
|
60
62
|
}
|
|
63
|
+
if (chunk &&
|
|
64
|
+
chunk.type === 'content_block_delta' &&
|
|
65
|
+
chunk.delta &&
|
|
66
|
+
chunk.delta.type === 'thinking_delta' &&
|
|
67
|
+
typeof chunk.delta.thinking === 'string') {
|
|
68
|
+
const thinkingByIndex = gapsFor(span).thinkingByIndex;
|
|
69
|
+
thinkingByIndex.set(chunk.index, (thinkingByIndex.get(chunk.index) ?? '') + chunk.delta.thinking);
|
|
70
|
+
}
|
|
61
71
|
}
|
|
62
72
|
catch {
|
|
63
73
|
// Never break the caller's stream over an enrichment failure.
|
|
@@ -124,6 +134,13 @@ class AnthropicStreamingAccumulationInstrumentor {
|
|
|
124
134
|
}
|
|
125
135
|
}
|
|
126
136
|
}
|
|
137
|
+
for (const [index, thinking] of gaps.thinkingByIndex) {
|
|
138
|
+
const block = Array.isArray(args.result.content) && args.result.content[index];
|
|
139
|
+
// Only fill the gap, as for the tool arguments above.
|
|
140
|
+
if (block && block.type === 'thinking' && !block.thinking) {
|
|
141
|
+
block.thinking = thinking;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
127
144
|
}
|
|
128
145
|
}
|
|
129
146
|
catch {
|
|
@@ -14,6 +14,13 @@
|
|
|
14
14
|
* It wraps the instrumentor instance's `_wrapPromise(span, promise, modelId)`,
|
|
15
15
|
* which resolves the send promise and enriches the span before ending it,
|
|
16
16
|
* letting us read the decoded body off the same response while the span is open.
|
|
17
|
+
*
|
|
18
|
+
* Claude on Bedrock also reports its extended-thinking count, a part of
|
|
19
|
+
* `output_tokens`, as `usage.output_tokens_details.thinking_tokens` in the
|
|
20
|
+
* response body, and in the closing `message_delta` chunk of a stream. The
|
|
21
|
+
* instrumentor drops it, so it is stamped as
|
|
22
|
+
* `gen_ai.usage.output_tokens_details.thinking_tokens`, the key the Anthropic
|
|
23
|
+
* complement stamps, matching the Python SDK.
|
|
17
24
|
*/
|
|
18
25
|
export declare class BedrockPromptCachingInstrumentor {
|
|
19
26
|
private _patched;
|
|
@@ -15,11 +15,19 @@
|
|
|
15
15
|
* It wraps the instrumentor instance's `_wrapPromise(span, promise, modelId)`,
|
|
16
16
|
* which resolves the send promise and enriches the span before ending it,
|
|
17
17
|
* letting us read the decoded body off the same response while the span is open.
|
|
18
|
+
*
|
|
19
|
+
* Claude on Bedrock also reports its extended-thinking count, a part of
|
|
20
|
+
* `output_tokens`, as `usage.output_tokens_details.thinking_tokens` in the
|
|
21
|
+
* response body, and in the closing `message_delta` chunk of a stream. The
|
|
22
|
+
* instrumentor drops it, so it is stamped as
|
|
23
|
+
* `gen_ai.usage.output_tokens_details.thinking_tokens`, the key the Anthropic
|
|
24
|
+
* complement stamps, matching the Python SDK.
|
|
18
25
|
*/
|
|
19
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
20
27
|
exports.BedrockPromptCachingInstrumentor = void 0;
|
|
21
28
|
const CACHE_READ_INPUT_TOKENS_ATTR = 'gen_ai.usage.cache_read_input_tokens';
|
|
22
29
|
const CACHE_CREATION_INPUT_TOKENS_ATTR = 'gen_ai.usage.cache_creation_input_tokens';
|
|
30
|
+
const THINKING_TOKENS_ATTR = 'gen_ai.usage.output_tokens_details.thinking_tokens';
|
|
23
31
|
// Stamp cache counts from a `usage` object onto the span: Nova's camelCase
|
|
24
32
|
// count fields, or Anthropic-on-Bedrock's native snake_case ones.
|
|
25
33
|
function _setCacheFromUsage(span, usage) {
|
|
@@ -34,6 +42,15 @@ function _setCacheFromUsage(span, usage) {
|
|
|
34
42
|
span.setAttribute(CACHE_CREATION_INPUT_TOKENS_ATTR, write);
|
|
35
43
|
}
|
|
36
44
|
}
|
|
45
|
+
// Stamp Claude's thinking count from a `usage` object onto the span.
|
|
46
|
+
function _setThinkingFromUsage(span, usage) {
|
|
47
|
+
if (!usage || !span || !span.isRecording || !span.isRecording())
|
|
48
|
+
return;
|
|
49
|
+
const thinking = usage.output_tokens_details?.thinking_tokens;
|
|
50
|
+
if (typeof thinking === 'number' && thinking) {
|
|
51
|
+
span.setAttribute(THINKING_TOKENS_ATTR, thinking);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
37
54
|
function _setCacheTokens(span, response) {
|
|
38
55
|
if (!span || !span.isRecording || !span.isRecording())
|
|
39
56
|
return;
|
|
@@ -45,7 +62,9 @@ function _setCacheTokens(span, response) {
|
|
|
45
62
|
if (!body || typeof body[Symbol.asyncIterator] === 'function')
|
|
46
63
|
return;
|
|
47
64
|
try {
|
|
48
|
-
|
|
65
|
+
const usage = JSON.parse(new TextDecoder().decode(body)).usage;
|
|
66
|
+
_setCacheFromUsage(span, usage);
|
|
67
|
+
_setThinkingFromUsage(span, usage);
|
|
49
68
|
}
|
|
50
69
|
catch {
|
|
51
70
|
// Never break span enrichment over a parse failure.
|
|
@@ -77,7 +96,8 @@ class BedrockPromptCachingInstrumentor {
|
|
|
77
96
|
// Streamed: the instrumentor hands every chunk to
|
|
78
97
|
// `_handleNovaStreamingMetadata(span, parsedResponse)` but never reads its
|
|
79
98
|
// cache counts: Nova's arrive in the trailing `metadata` chunk, Anthropic's
|
|
80
|
-
// in the opening `message_start` one.
|
|
99
|
+
// in the opening `message_start` one. Claude's thinking count arrives in its
|
|
100
|
+
// closing `message_delta` one.
|
|
81
101
|
const metadataOriginal = instrumentation._handleNovaStreamingMetadata;
|
|
82
102
|
if (typeof metadataOriginal === 'function') {
|
|
83
103
|
instrumentation._handleNovaStreamingMetadata = function handleMetadataWithCache(span, parsedResponse) {
|
|
@@ -86,6 +106,9 @@ class BedrockPromptCachingInstrumentor {
|
|
|
86
106
|
? parsedResponse.message?.usage
|
|
87
107
|
: parsedResponse?.metadata?.usage;
|
|
88
108
|
_setCacheFromUsage(span, usage);
|
|
109
|
+
if (parsedResponse?.type === 'message_delta') {
|
|
110
|
+
_setThinkingFromUsage(span, parsedResponse.usage);
|
|
111
|
+
}
|
|
89
112
|
}
|
|
90
113
|
catch {
|
|
91
114
|
// Never break the caller's stream over an enrichment failure.
|
|
@@ -12,24 +12,31 @@
|
|
|
12
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
13
|
exports.BedrockInputMessageContentInstrumentor = void 0;
|
|
14
14
|
const _MEDIA_BLOCK_TYPES = ['image', 'document', 'video', 'audio'];
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
15
|
+
function _buildBlobPart(mimeType, base64Bytes) {
|
|
16
|
+
return typeof base64Bytes === 'string'
|
|
17
|
+
? { type: 'blob', mime_type: mimeType, content: base64Bytes }
|
|
18
|
+
: { type: 'blob', mime_type: mimeType };
|
|
19
|
+
}
|
|
20
|
+
// A blob part holding the bytes of a media content block, null when the block
|
|
21
|
+
// is not media. Anthropic-on-Bedrock names the media under `type` and carries
|
|
22
|
+
// its base64 in `source.data` and its mime type in `source.media_type`; Nova puts
|
|
23
|
+
// the media under its own key
|
|
24
|
+
// (`{"image": {"format": "png", "source": {"bytes": ...}}}`) and states no mime
|
|
25
|
+
// type.
|
|
26
|
+
function _getMediaBlobPart(block) {
|
|
20
27
|
if (_MEDIA_BLOCK_TYPES.includes(block.type)) {
|
|
21
|
-
const source = block.source;
|
|
22
|
-
|
|
23
|
-
return { type: 'blob', mime_type: String(mediaType || '') };
|
|
28
|
+
const source = block.source && typeof block.source === 'object' ? block.source : {};
|
|
29
|
+
return _buildBlobPart(String(source.media_type || ''), source.data);
|
|
24
30
|
}
|
|
25
|
-
|
|
26
|
-
|
|
31
|
+
const novaMedia = _MEDIA_BLOCK_TYPES.find((media) => block[media] && typeof block[media] === 'object');
|
|
32
|
+
if (novaMedia) {
|
|
33
|
+
const source = block[novaMedia].source;
|
|
34
|
+
return _buildBlobPart('', source && typeof source === 'object' ? source.bytes : undefined);
|
|
27
35
|
}
|
|
28
36
|
return null;
|
|
29
37
|
}
|
|
30
|
-
// Text blocks keep their text; media blocks become
|
|
31
|
-
//
|
|
32
|
-
// that the call carried content the stored text does not.
|
|
38
|
+
// Text blocks keep their text; media blocks become blob parts holding their
|
|
39
|
+
// bytes.
|
|
33
40
|
function _contentToParts(content) {
|
|
34
41
|
if (typeof content === 'string') {
|
|
35
42
|
return [{ type: 'text', content }];
|
|
@@ -44,9 +51,9 @@ function _contentToParts(content) {
|
|
|
44
51
|
parts.push({ type: 'text', content: block.text });
|
|
45
52
|
continue;
|
|
46
53
|
}
|
|
47
|
-
const
|
|
48
|
-
if (
|
|
49
|
-
parts.push(
|
|
54
|
+
const mediaBlobPart = _getMediaBlobPart(block);
|
|
55
|
+
if (mediaBlobPart !== null)
|
|
56
|
+
parts.push(mediaBlobPart);
|
|
50
57
|
}
|
|
51
58
|
return parts;
|
|
52
59
|
}
|
|
@@ -19,6 +19,12 @@
|
|
|
19
19
|
* a forced tool call streams into an empty message carrying only
|
|
20
20
|
* `finish_reason: "tool_call"`. Upstream accumulates tool blocks on the Converse
|
|
21
21
|
* API path only, never on InvokeModelWithResponseStream.
|
|
22
|
+
*
|
|
23
|
+
* Anthropic's extended thinking streams as `thinking_delta`s, which upstream
|
|
24
|
+
* drops too, so they are accumulated into a `reasoning` part written before the
|
|
25
|
+
* text, the part a non-streamed answer gets (see
|
|
26
|
+
* `get_reasoning_output_telemetry.ts`). A call that only thinks, stopped by its
|
|
27
|
+
* token cap before any text, otherwise records an empty message.
|
|
22
28
|
*/
|
|
23
29
|
export declare class BedrockStreamingOutputMessageInstrumentor {
|
|
24
30
|
private _patched;
|
|
@@ -20,6 +20,12 @@
|
|
|
20
20
|
* a forced tool call streams into an empty message carrying only
|
|
21
21
|
* `finish_reason: "tool_call"`. Upstream accumulates tool blocks on the Converse
|
|
22
22
|
* API path only, never on InvokeModelWithResponseStream.
|
|
23
|
+
*
|
|
24
|
+
* Anthropic's extended thinking streams as `thinking_delta`s, which upstream
|
|
25
|
+
* drops too, so they are accumulated into a `reasoning` part written before the
|
|
26
|
+
* text, the part a non-streamed answer gets (see
|
|
27
|
+
* `get_reasoning_output_telemetry.ts`). A call that only thinks, stopped by its
|
|
28
|
+
* token cap before any text, otherwise records an empty message.
|
|
23
29
|
*/
|
|
24
30
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
31
|
exports.BedrockStreamingOutputMessageInstrumentor = void 0;
|
|
@@ -58,7 +64,8 @@ function _accumulateToolCall(state, parsedResponse) {
|
|
|
58
64
|
class BedrockStreamingOutputMessageInstrumentor {
|
|
59
65
|
// [instrumentation, methodName, originalFunction] for uninstrument().
|
|
60
66
|
_patched = [];
|
|
61
|
-
// span -> {text, stopReason} accumulated across the
|
|
67
|
+
// span -> {reasoning, text, stopReason, toolCalls} accumulated across the
|
|
68
|
+
// stream's chunks.
|
|
62
69
|
_streamStateBySpan = new WeakMap();
|
|
63
70
|
// No-op (and never throws) if the instrumentor's internals have changed shape —
|
|
64
71
|
// the worst case is an empty streamed output message, never a crash.
|
|
@@ -72,6 +79,7 @@ class BedrockStreamingOutputMessageInstrumentor {
|
|
|
72
79
|
instrumentation._handleNovaStreamingMetadata = function handleWithOutputMessage(span, parsedResponse) {
|
|
73
80
|
try {
|
|
74
81
|
const state = streamStateBySpan.get(span) || {
|
|
82
|
+
reasoning: '',
|
|
75
83
|
text: '',
|
|
76
84
|
stopReason: null,
|
|
77
85
|
toolCalls: [],
|
|
@@ -80,6 +88,12 @@ class BedrockStreamingOutputMessageInstrumentor {
|
|
|
80
88
|
(parsedResponse?.type === 'content_block_delta' ? parsedResponse.delta?.text : undefined);
|
|
81
89
|
if (typeof delta === 'string')
|
|
82
90
|
state.text += delta;
|
|
91
|
+
const thinkingDelta = parsedResponse?.type === 'content_block_delta' &&
|
|
92
|
+
parsedResponse.delta?.type === 'thinking_delta'
|
|
93
|
+
? parsedResponse.delta.thinking
|
|
94
|
+
: undefined;
|
|
95
|
+
if (typeof thinkingDelta === 'string')
|
|
96
|
+
state.reasoning += thinkingDelta;
|
|
83
97
|
const stopReason = parsedResponse?.messageStop?.stopReason ??
|
|
84
98
|
(parsedResponse?.type === 'message_delta'
|
|
85
99
|
? parsedResponse.delta?.stop_reason
|
|
@@ -89,8 +103,10 @@ class BedrockStreamingOutputMessageInstrumentor {
|
|
|
89
103
|
_accumulateToolCall(state, parsedResponse);
|
|
90
104
|
streamStateBySpan.set(span, state);
|
|
91
105
|
const isLastChunk = parsedResponse?.metadata || parsedResponse?.type === 'message_stop';
|
|
92
|
-
if (isLastChunk && (state.text || state.toolCalls.length)) {
|
|
106
|
+
if (isLastChunk && (state.reasoning || state.text || state.toolCalls.length)) {
|
|
93
107
|
const parts = [];
|
|
108
|
+
if (state.reasoning)
|
|
109
|
+
parts.push({ type: 'reasoning', content: state.reasoning });
|
|
94
110
|
if (state.text)
|
|
95
111
|
parts.push({ type: 'text', content: state.text });
|
|
96
112
|
for (const toolCall of state.toolCalls) {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.BedrockTelemetryInstrumentor = void 0;
|
|
18
18
|
const get_cache_config_telemetry_1 = require("./get_cache_config_telemetry");
|
|
19
|
-
const
|
|
19
|
+
const get_cache_and_reasoning_telemetry_1 = require("./get_cache_and_reasoning_telemetry");
|
|
20
20
|
const get_input_message_content_telemetry_1 = require("./get_input_message_content_telemetry");
|
|
21
21
|
const get_streaming_output_message_telemetry_1 = require("./get_streaming_output_message_telemetry");
|
|
22
22
|
const get_model_id_telemetry_1 = require("./get_model_id_telemetry");
|
|
@@ -28,7 +28,7 @@ class BedrockTelemetryInstrumentor {
|
|
|
28
28
|
_subInstrumentors;
|
|
29
29
|
constructor() {
|
|
30
30
|
this._subInstrumentors = [
|
|
31
|
-
new
|
|
31
|
+
new get_cache_and_reasoning_telemetry_1.BedrockPromptCachingInstrumentor(),
|
|
32
32
|
new get_cache_config_telemetry_1.BedrockCacheConfigInstrumentor(),
|
|
33
33
|
new get_input_message_content_telemetry_1.BedrockInputMessageContentInstrumentor(),
|
|
34
34
|
new get_streaming_output_message_telemetry_1.BedrockStreamingOutputMessageInstrumentor(),
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
* Input message content for Cohere v2 chat spans.
|
|
3
3
|
*
|
|
4
4
|
* Mirror of the Python complement's message shaping: a v2 request's messages in
|
|
5
|
-
* the standard role+parts shape, with
|
|
6
|
-
*
|
|
7
|
-
* still see that the call carried content the stored text does not.
|
|
5
|
+
* the standard role+parts shape, with an inline image as a `blob` part holding
|
|
6
|
+
* its bytes, which the ingestion API moves to S3.
|
|
8
7
|
*
|
|
9
8
|
* Python wraps the OpenLLMetry instrumentor to stamp this (that instrumentor
|
|
10
9
|
* covers v2 but records its messages in a shape OTel rejects). The Node
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
* Input message content for Cohere v2 chat spans.
|
|
4
4
|
*
|
|
5
5
|
* Mirror of the Python complement's message shaping: a v2 request's messages in
|
|
6
|
-
* the standard role+parts shape, with
|
|
7
|
-
*
|
|
8
|
-
* still see that the call carried content the stored text does not.
|
|
6
|
+
* the standard role+parts shape, with an inline image as a `blob` part holding
|
|
7
|
+
* its bytes, which the ingestion API moves to S3.
|
|
9
8
|
*
|
|
10
9
|
* Python wraps the OpenLLMetry instrumentor to stamp this (that instrumentor
|
|
11
10
|
* covers v2 but records its messages in a shape OTel rejects). The Node
|
|
@@ -22,8 +21,8 @@ function asObject(value) {
|
|
|
22
21
|
: null;
|
|
23
22
|
}
|
|
24
23
|
/**
|
|
25
|
-
* An image block as a part: an inline data URI
|
|
26
|
-
* hosted image keeps its url
|
|
24
|
+
* An image block as a part: an inline data URI becomes a blob holding its
|
|
25
|
+
* bytes, a hosted image keeps its url.
|
|
27
26
|
*/
|
|
28
27
|
function getImagePart(block) {
|
|
29
28
|
// The JS client spells the key imageUrl; the wire (and the Python client)
|
|
@@ -31,14 +30,11 @@ function getImagePart(block) {
|
|
|
31
30
|
const rawImageUrl = block.imageUrl ?? block.image_url;
|
|
32
31
|
const imageUrl = asObject(rawImageUrl);
|
|
33
32
|
const url = String((imageUrl ? imageUrl.url : rawImageUrl) ?? '');
|
|
34
|
-
|
|
35
|
-
if (mimeType === null)
|
|
36
|
-
return { type: 'uri', uri: url };
|
|
37
|
-
return { type: 'blob', mime_type: mimeType };
|
|
33
|
+
return (0, get_data_uri_mime_type_1.getDataUriBlobPart)(url) ?? { type: 'uri', uri: url };
|
|
38
34
|
}
|
|
39
35
|
/**
|
|
40
36
|
* A v2 message's `content` as normalized parts. A plain string is one text
|
|
41
|
-
* part; a list keeps its text, turns images into
|
|
37
|
+
* part; a list keeps its text, turns images into blob parts, and keeps
|
|
42
38
|
* any other block under its own type - naming it is what lets the analyses tell
|
|
43
39
|
* a text-only prompt from one whose content was not stored.
|
|
44
40
|
*/
|
|
@@ -46,9 +46,13 @@ function _contentToMessage(content) {
|
|
|
46
46
|
parts.push({ type: 'text', content: rawPart.text });
|
|
47
47
|
}
|
|
48
48
|
else if (rawPart && rawPart.inlineData) {
|
|
49
|
-
// Non-text parts (e.g. audio) become a
|
|
50
|
-
//
|
|
51
|
-
parts.push({
|
|
49
|
+
// Non-text parts (e.g. audio) become a blob part holding the base64
|
|
50
|
+
// bytes, which the ingestion API moves to S3.
|
|
51
|
+
parts.push({
|
|
52
|
+
type: 'blob',
|
|
53
|
+
mime_type: rawPart.inlineData.mimeType || '',
|
|
54
|
+
content: rawPart.inlineData.data,
|
|
55
|
+
});
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
58
|
return { role, parts };
|
|
@@ -34,10 +34,16 @@
|
|
|
34
34
|
* under a bare block type with no mime type, which is repaired (see
|
|
35
35
|
* `get_input_message_content_telemetry.ts`).
|
|
36
36
|
* - On the response, it only forwards the flat `prompt_tokens` /
|
|
37
|
-
* `completion_tokens` counts, so prompt-cache reads
|
|
38
|
-
* `usage.prompt_tokens_details.cached_tokens`
|
|
39
|
-
* (
|
|
40
|
-
* `
|
|
37
|
+
* `completion_tokens` counts, so prompt-cache reads and the reasoning count
|
|
38
|
+
* never reach the span: `usage.prompt_tokens_details.cached_tokens` and
|
|
39
|
+
* `usage.completion_tokens_details.reasoning_tokens` (a part of
|
|
40
|
+
* `completion_tokens`) are read off the response body (or off the streamed
|
|
41
|
+
* chunks as they pass) and stamped as `gen_ai.usage.cache_read_input_tokens`
|
|
42
|
+
* and `gen_ai.usage.output_tokens_details.reasoning_tokens`.
|
|
43
|
+
* - On a stream, its accumulation of the chunks drops the `thinking` blocks, so
|
|
44
|
+
* a streamed thinking call records no reasoning part: the thinking is read off
|
|
45
|
+
* the streamed chunks as they pass and added to `gen_ai.output.messages` as it
|
|
46
|
+
* is set (see `get_streamed_reasoning_telemetry.ts`).
|
|
41
47
|
*
|
|
42
48
|
* Mistral's models on Vertex AI, called through `MistralGCP`, get the same
|
|
43
49
|
* tracing from `get_vertex_telemetry.ts`.
|
|
@@ -35,10 +35,16 @@
|
|
|
35
35
|
* under a bare block type with no mime type, which is repaired (see
|
|
36
36
|
* `get_input_message_content_telemetry.ts`).
|
|
37
37
|
* - On the response, it only forwards the flat `prompt_tokens` /
|
|
38
|
-
* `completion_tokens` counts, so prompt-cache reads
|
|
39
|
-
* `usage.prompt_tokens_details.cached_tokens`
|
|
40
|
-
* (
|
|
41
|
-
* `
|
|
38
|
+
* `completion_tokens` counts, so prompt-cache reads and the reasoning count
|
|
39
|
+
* never reach the span: `usage.prompt_tokens_details.cached_tokens` and
|
|
40
|
+
* `usage.completion_tokens_details.reasoning_tokens` (a part of
|
|
41
|
+
* `completion_tokens`) are read off the response body (or off the streamed
|
|
42
|
+
* chunks as they pass) and stamped as `gen_ai.usage.cache_read_input_tokens`
|
|
43
|
+
* and `gen_ai.usage.output_tokens_details.reasoning_tokens`.
|
|
44
|
+
* - On a stream, its accumulation of the chunks drops the `thinking` blocks, so
|
|
45
|
+
* a streamed thinking call records no reasoning part: the thinking is read off
|
|
46
|
+
* the streamed chunks as they pass and added to `gen_ai.output.messages` as it
|
|
47
|
+
* is set (see `get_streamed_reasoning_telemetry.ts`).
|
|
42
48
|
*
|
|
43
49
|
* Mistral's models on Vertex AI, called through `MistralGCP`, get the same
|
|
44
50
|
* tracing from `get_vertex_telemetry.ts`.
|
|
@@ -51,6 +57,7 @@ const logger_1 = require("../../logger");
|
|
|
51
57
|
const span_processor_1 = require("../../otel/span-processor");
|
|
52
58
|
const get_structured_output_schema_telemetry_1 = require("../shared/get_structured_output_schema_telemetry");
|
|
53
59
|
const get_input_message_content_telemetry_1 = require("./get_input_message_content_telemetry");
|
|
60
|
+
const get_streamed_reasoning_telemetry_1 = require("./get_streamed_reasoning_telemetry");
|
|
54
61
|
const get_structured_output_schema_telemetry_2 = require("./get_structured_output_schema_telemetry");
|
|
55
62
|
const get_vertex_telemetry_1 = require("./get_vertex_telemetry");
|
|
56
63
|
exports.MISTRAL_SDK_TELEMETRY_ENV = 'MISTRAL_SDK_TELEMETRY';
|
|
@@ -60,6 +67,7 @@ const CACHE_KEY_ATTR = 'weflayr.cache.key';
|
|
|
60
67
|
const REASONING_EFFORT_ATTR = 'gen_ai.request.reasoning_effort';
|
|
61
68
|
const EMBEDDINGS_INPUT_ATTR = 'weflayr.embeddings.input';
|
|
62
69
|
const CACHE_READ_INPUT_TOKENS_ATTR = 'gen_ai.usage.cache_read_input_tokens';
|
|
70
|
+
const REASONING_TOKENS_ATTR = 'gen_ai.usage.output_tokens_details.reasoning_tokens';
|
|
63
71
|
const OUTPUT_MESSAGES_ATTR = 'gen_ai.output.messages';
|
|
64
72
|
const FINISH_REASONS_ATTR = 'gen_ai.response.finish_reasons';
|
|
65
73
|
// Mistral's API vocabulary to the OTel GenAI finish reason (the value the
|
|
@@ -116,6 +124,23 @@ function toOtelOutputMessages(outputMessagesJson) {
|
|
|
116
124
|
}
|
|
117
125
|
return JSON.stringify(messages);
|
|
118
126
|
}
|
|
127
|
+
// Each streamed call's thinking by choice index, read off its chunks as they
|
|
128
|
+
// pass, until the tracing writes its output messages.
|
|
129
|
+
const streamedThinkingBySpan = new WeakMap();
|
|
130
|
+
/** `gen_ai.output.messages` with the call's streamed thinking added; unchanged
|
|
131
|
+
* for a call that streamed none, or when it cannot be added. */
|
|
132
|
+
function addSpanStreamedThinking(span, outputMessagesJson) {
|
|
133
|
+
const thinkingByChoice = streamedThinkingBySpan.get(span);
|
|
134
|
+
if (!thinkingByChoice)
|
|
135
|
+
return outputMessagesJson;
|
|
136
|
+
try {
|
|
137
|
+
return (0, get_streamed_reasoning_telemetry_1.addStreamedThinkingToOutputMessages)(outputMessagesJson, thinkingByChoice);
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
(0, logger_1.debug)('mistral: streamed thinking not recorded: %s', err?.message || err);
|
|
141
|
+
return outputMessagesJson;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
119
144
|
function contentCaptureEnabled() {
|
|
120
145
|
// Read at span time (not instrument time) so a later `weflayr.setup()`
|
|
121
146
|
// toggling captureMessageContent takes effect. Defaults to off, matching the
|
|
@@ -131,7 +156,7 @@ function shapeAttributesAsTheyAreSet(span) {
|
|
|
131
156
|
if (!contentCaptureEnabled() && MESSAGE_CONTENT_ATTRIBUTES.includes(key))
|
|
132
157
|
return span;
|
|
133
158
|
if (key === OUTPUT_MESSAGES_ATTR && typeof value === 'string') {
|
|
134
|
-
return originalSetAttribute(key, toOtelOutputMessages(value));
|
|
159
|
+
return originalSetAttribute(key, toOtelOutputMessages(addSpanStreamedThinking(span, value)));
|
|
135
160
|
}
|
|
136
161
|
if (key === FINISH_REASONS_ATTR && Array.isArray(value)) {
|
|
137
162
|
return originalSetAttribute(key, value.map(toOtelFinishReason));
|
|
@@ -174,18 +199,24 @@ function setRequestBodyTelemetry(span, requestBodyJson) {
|
|
|
174
199
|
span.setAttribute(EMBEDDINGS_INPUT_ATTR, JSON.stringify(embeddedTexts));
|
|
175
200
|
}
|
|
176
201
|
}
|
|
177
|
-
function
|
|
202
|
+
function setTokenDetails(span, responseData) {
|
|
178
203
|
const usage = responseData?.usage;
|
|
179
204
|
const cachedTokens = usage?.prompt_tokens_details?.cached_tokens;
|
|
180
205
|
if (cachedTokens)
|
|
181
206
|
span.setAttribute(CACHE_READ_INPUT_TOKENS_ATTR, cachedTokens);
|
|
207
|
+
const reasoningTokens = usage?.completion_tokens_details?.reasoning_tokens;
|
|
208
|
+
if (reasoningTokens)
|
|
209
|
+
span.setAttribute(REASONING_TOKENS_ATTR, reasoningTokens);
|
|
182
210
|
}
|
|
183
211
|
/**
|
|
184
|
-
* The streamed response, unchanged, with every chunk's usage
|
|
185
|
-
* passes: the tracing hook reads the same stream, and ends the span only
|
|
186
|
-
* it is exhausted, so the cache-read
|
|
212
|
+
* The streamed response, unchanged, with every chunk's usage and thinking read
|
|
213
|
+
* as it passes: the tracing hook reads the same stream, and ends the span only
|
|
214
|
+
* once it is exhausted, so the cache-read and reasoning counts are on the span, and the
|
|
215
|
+
* thinking ready for its output messages, by then.
|
|
187
216
|
*/
|
|
188
|
-
function
|
|
217
|
+
function readTokenDetailsAndThinkingFromStream(span, response) {
|
|
218
|
+
const thinkingByChoice = new Map();
|
|
219
|
+
streamedThinkingBySpan.set(span, thinkingByChoice);
|
|
189
220
|
const reader = response.body.getReader();
|
|
190
221
|
const decoder = new TextDecoder();
|
|
191
222
|
let unfinishedLine = '';
|
|
@@ -197,10 +228,12 @@ function readCachedTokensFromStream(span, response) {
|
|
|
197
228
|
if (!data.startsWith(SERVER_SENT_EVENT_DATA_PREFIX))
|
|
198
229
|
continue;
|
|
199
230
|
try {
|
|
200
|
-
|
|
231
|
+
const chunk = JSON.parse(data.slice(SERVER_SENT_EVENT_DATA_PREFIX.length));
|
|
232
|
+
setTokenDetails(span, chunk);
|
|
233
|
+
(0, get_streamed_reasoning_telemetry_1.addStreamedThinking)(chunk, thinkingByChoice);
|
|
201
234
|
}
|
|
202
235
|
catch {
|
|
203
|
-
// "[DONE]" and any line that is not a JSON chunk carry no usage.
|
|
236
|
+
// "[DONE]" and any line that is not a JSON chunk carry no usage or thinking.
|
|
204
237
|
}
|
|
205
238
|
}
|
|
206
239
|
};
|
|
@@ -235,10 +268,10 @@ async function observeCachedTokens(span, response) {
|
|
|
235
268
|
return response;
|
|
236
269
|
const contentType = (response.headers.get('content-type') || '').toLowerCase();
|
|
237
270
|
if (contentType.includes('text/event-stream') && response.body) {
|
|
238
|
-
return
|
|
271
|
+
return readTokenDetailsAndThinkingFromStream(span, response);
|
|
239
272
|
}
|
|
240
273
|
if (contentType.includes('json')) {
|
|
241
|
-
|
|
274
|
+
setTokenDetails(span, await response.clone().json());
|
|
242
275
|
}
|
|
243
276
|
return response;
|
|
244
277
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* `gen_ai.input.messages` as a bare `{"type": "<block type>"}` (the fallback
|
|
7
7
|
* branch of its `formatting.ts`), so a PDF sent as `document_url` is stored
|
|
8
8
|
* under a name with no mime type, where every other provider stores the
|
|
9
|
-
*
|
|
9
|
+
* `{"type": "blob", "mime_type": "application/pdf", "content": <base64>}`.
|
|
10
10
|
*
|
|
11
11
|
* The mime type it lacks is in the request the caller sent: the block's url is
|
|
12
12
|
* a `data:` URI holding the bytes. This complement walks the parts the tracing
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* `gen_ai.input.messages` as a bare `{"type": "<block type>"}` (the fallback
|
|
8
8
|
* branch of its `formatting.ts`), so a PDF sent as `document_url` is stored
|
|
9
9
|
* under a name with no mime type, where every other provider stores the
|
|
10
|
-
*
|
|
10
|
+
* `{"type": "blob", "mime_type": "application/pdf", "content": <base64>}`.
|
|
11
11
|
*
|
|
12
12
|
* The mime type it lacks is in the request the caller sent: the block's url is
|
|
13
13
|
* a `data:` URI holding the bytes. This complement walks the parts the tracing
|
|
@@ -38,20 +38,17 @@ function getBlockUrl(block) {
|
|
|
38
38
|
return String(url ?? '');
|
|
39
39
|
}
|
|
40
40
|
/**
|
|
41
|
-
* The
|
|
42
|
-
*
|
|
41
|
+
* The blob part for a content block carrying inline bytes, or null for a block
|
|
42
|
+
* that carries none.
|
|
43
43
|
*
|
|
44
44
|
* e.g. `{"type": "document_url", "document_url": "data:application/pdf;base64,AAA"}`
|
|
45
|
-
* -> `{"type": "blob", "mime_type": "application/pdf"}`.
|
|
45
|
+
* -> `{"type": "blob", "mime_type": "application/pdf", "content": "AAA"}`.
|
|
46
46
|
*/
|
|
47
47
|
function getMediaStub(rawBlock) {
|
|
48
48
|
const block = asObject(rawBlock);
|
|
49
49
|
if (block === null)
|
|
50
50
|
return null;
|
|
51
|
-
|
|
52
|
-
if (mimeType === null)
|
|
53
|
-
return null;
|
|
54
|
-
return { type: 'blob', mime_type: mimeType };
|
|
51
|
+
return (0, get_data_uri_mime_type_1.getDataUriBlobPart)(getBlockUrl(block));
|
|
55
52
|
}
|
|
56
53
|
/** True for the `{"type": "<block type>"}` the tracing falls back to on a
|
|
57
54
|
* block it does not shape: a type and nothing else. */
|