weflayr 0.20.5 → 0.21.0

package/tests/instrumentation/bedrock/get_cache_telemetry.test.js

@@ -0,0 +1,102 @@
+'use strict';
+
+// Tests for the Bedrock cached-prompt token complement. It wraps the primary
+// instrumentor's `_wrapPromise(span, promise, modelId)` to read Nova's cache-token
+// counts off the decoded InvokeModel response body and stamp them as
+// gen_ai.usage.cache_* attributes.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const {
+  BedrockPromptCachingInstrumentor,
+} = require('../../../src/instrumentation/bedrock/get_cache_telemetry');
+
+function fakeSpan() {
+  const attrs = {};
+  return {
+    attrs,
+    isRecording: () => true,
+    setAttribute(key, value) {
+      attrs[key] = value;
+    },
+  };
+}
+
+function encode(obj) {
+  return new TextEncoder().encode(JSON.stringify(obj));
+}
+
+// Pass-through original: resolve to whatever promise it was handed.
+function passthroughInstrumentation() {
+  return { _wrapPromise: (_span, promise) => Promise.resolve(promise) };
+}
+
+test('stamps cache read/write from a buffered InvokeModel body', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  const body = encode({
+    usage: { cacheReadInputTokenCount: 100, cacheWriteInputTokenCount: 1300 },
+  });
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'amazon.nova-micro-v1:0');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 100);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], 1300);
+});
+
+test('sets nothing when cache counts are zero', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  const body = encode({ usage: { cacheReadInputTokenCount: 0, cacheWriteInputTokenCount: 0 } });
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'm');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+});
+
+test('skips a streamed (async-iterable) body without consuming it', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  let consumed = false;
+  const body = (async function* () {
+    consumed = true;
+    yield {};
+  })();
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'm');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(consumed, false);
+});
+
+test('stamps cache from the streamed metadata chunk and calls the original', () => {
+  const calls = [];
+  const instrumentation = {
+    _handleNovaStreamingMetadata(span, parsed) {
+      calls.push(parsed);
+    },
+  };
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._handleNovaStreamingMetadata(span, {
+    metadata: {
+      usage: { inputTokens: 3, cacheReadInputTokenCount: 1458, cacheWriteInputTokenCount: 0 },
+    },
+  });
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 1458);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+  assert.equal(calls.length, 1);
+});
+
+test('is a no-op when the instrumentor has no _wrapPromise', () => {
+  const instrumentation = {};
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+  assert.equal(instrumentation._wrapPromise, undefined);
+});

package/tests/instrumentation/cohere/get_chat_telemetry.test.js

@@ -0,0 +1,148 @@
+'use strict';
+
+// Tests for the Cohere chat token-usage enrichment complement.
+//
+// The cohere-ai v1 `CohereClient` is stubbed with an internal `__chat` method
+// that mirrors the SDK shape — resolving to `{ data: { meta: { billedUnits } } }`
+// — so the complement can be exercised without network access or the dependency
+// installed. The complement enriches whichever span is active when `__chat` runs
+// (in production, the span traceloop opens around the public `chat`), so the
+// tests start a span and make it active before invoking `__chat`.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Module = require('node:module');
+
+const { trace, context } = require('@opentelemetry/api');
+const {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} = require('@opentelemetry/sdk-trace-base');
+const { AsyncLocalStorageContextManager } = require('@opentelemetry/context-async-hooks');
+
+const {
+  CohereChatTokenUsageInstrumentor,
+} = require('../../../src/instrumentation/cohere/get_chat_telemetry');
+
+const CLIENT_MODULE = 'cohere-ai/Client';
+
+const exporter = new InMemorySpanExporter();
+trace.setGlobalTracerProvider(
+  new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
+);
+context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+const tracer = trace.getTracer('test');
+
+class StubCohereClient {
+  __chat(request = {}) {
+    if (request.model === 'boom') return Promise.reject(new Error('api error'));
+    return Promise.resolve({
+      data: { text: 'hi', meta: { billedUnits: { inputTokens: 7, outputTokens: 3 } } },
+    });
+  }
+}
+
+function instrumented() {
+  const stubs = { [CLIENT_MODULE]: { CohereClient: StubCohereClient } };
+  const origResolve = Module._resolveFilename;
+  Module._resolveFilename = function patched(request, ...rest) {
+    if (request in stubs) return request;
+    return origResolve.call(this, request, ...rest);
+  };
+  for (const [moduleId, exports] of Object.entries(stubs)) {
+    require.cache[moduleId] = { id: moduleId, filename: moduleId, loaded: true, exports };
+  }
+
+  const instrumentor = new CohereChatTokenUsageInstrumentor();
+  instrumentor.instrument();
+
+  return () => {
+    instrumentor.uninstrument();
+    Module._resolveFilename = origResolve;
+    for (const moduleId of Object.keys(stubs)) delete require.cache[moduleId];
+    exporter.reset();
+  };
+}
+
+// Runs `__chat` inside an active span, mirroring the context traceloop sets up
+// around the public `chat`, and returns the finished span.
+async function callChatWithinSpan(client, request) {
+  const span = tracer.startSpan('cohere.chat');
+  await context.with(trace.setSpan(context.active(), span), () => client.__chat(request));
+  span.end();
+  return exporter.getFinishedSpans().find((finished) => finished.name === 'cohere.chat');
+}
+
+test('enriches the active chat span with input and output tokens', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    const span = await callChatWithinSpan(client, { model: 'command-r-08-2024', message: 'hi' });
+
+    assert.equal(span.attributes['gen_ai.usage.input_tokens'], 7);
+    assert.equal(span.attributes['gen_ai.usage.output_tokens'], 3);
+  } finally {
+    teardown();
+  }
+});
+
+test('leaves the response unchanged', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    let result;
+    const span = tracer.startSpan('cohere.chat');
+    await context.with(trace.setSpan(context.active(), span), async () => {
+      result = await client.__chat({ model: 'command-r-08-2024', message: 'hi' });
+    });
+    span.end();
+
+    assert.equal(result.data.text, 'hi');
+    assert.equal(result.data.meta.billedUnits.inputTokens, 7);
+  } finally {
+    teardown();
+  }
+});
+
+test('does nothing when no span is active', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    // No active span — must not throw, just pass the response through.
+    const result = await client.__chat({ model: 'command-r-08-2024', message: 'hi' });
+    assert.equal(result.data.meta.billedUnits.outputTokens, 3);
+    assert.equal(exporter.getFinishedSpans().length, 0);
+  } finally {
+    teardown();
+  }
+});
+
+test('propagates errors without enriching the span', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    const span = tracer.startSpan('cohere.chat');
+    await assert.rejects(
+      () =>
+        context.with(trace.setSpan(context.active(), span), () =>
+          client.__chat({ model: 'boom', message: 'x' })
+        ),
+      /api error/
+    );
+    span.end();
+
+    const finished = exporter.getFinishedSpans().find((each) => each.name === 'cohere.chat');
+    assert.equal(finished.attributes['gen_ai.usage.input_tokens'], undefined);
+  } finally {
+    teardown();
+  }
+});
+
+test('uninstrument restores the original method', async () => {
+  const teardown = instrumented();
+  const { CohereClient } = require(CLIENT_MODULE);
+  const wrapped = CohereClient.prototype.__chat;
+  teardown();
+  assert.notEqual(CohereClient.prototype.__chat, wrapped);
+});

package/tests/instrumentation/cohere/get_embeddings_telemetry.test.js

@@ -0,0 +1,129 @@
+'use strict';
+
+// Tests for the bundled Cohere embeddings instrumentor.
+//
+// The cohere-ai v1 `CohereClient` and v2 `V2Client` classes are stubbed with an
+// `embed` method that mirrors the SDK shape — resolving to a response carrying
+// `meta.billedUnits.inputTokens` — so the instrumentor can be exercised without
+// network access or the dependency installed.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Module = require('node:module');
+
+const { trace, SpanStatusCode } = require('@opentelemetry/api');
+const {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} = require('@opentelemetry/sdk-trace-base');
+
+const {
+  CohereEmbeddingsInstrumentor,
+} = require('../../../src/instrumentation/cohere/get_embeddings_telemetry');
+
+const V1_MODULE = 'cohere-ai/Client';
+const V2_MODULE = 'cohere-ai/api/resources/v2/client/Client';
+
+const exporter = new InMemorySpanExporter();
+trace.setGlobalTracerProvider(
+  new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
+);
+
+class StubCohereClient {
+  embed(request = {}) {
+    if (request.model === 'boom') return Promise.reject(new Error('api error'));
+    return Promise.resolve({ embeddings: [[0.1]], meta: { billedUnits: { inputTokens: 8 } } });
+  }
+}
+
+class StubV2Client {
+  embed(_request = {}) {
+    return Promise.resolve({
+      embeddings: { float: [[0.2]] },
+      meta: { billedUnits: { inputTokens: 5 } },
+    });
+  }
+}
+
+function instrumented() {
+  const stubs = {
+    [V1_MODULE]: { CohereClient: StubCohereClient },
+    [V2_MODULE]: { V2Client: StubV2Client },
+  };
+  const origResolve = Module._resolveFilename;
+  Module._resolveFilename = function patched(request, ...rest) {
+    if (request in stubs) return request;
+    return origResolve.call(this, request, ...rest);
+  };
+  for (const [moduleId, exports] of Object.entries(stubs)) {
+    require.cache[moduleId] = { id: moduleId, filename: moduleId, loaded: true, exports };
+  }
+
+  const instrumentor = new CohereEmbeddingsInstrumentor();
+  instrumentor.instrument();
+
+  return () => {
+    instrumentor.uninstrument();
+    Module._resolveFilename = origResolve;
+    for (const moduleId of Object.keys(stubs)) delete require.cache[moduleId];
+    exporter.reset();
+  };
+}
+
+test('v1 embed emits genai span with input tokens', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    const result = await client.embed({ model: 'embed-english-v3.0', texts: ['hi'] });
+    assert.equal(result.meta.billedUnits.inputTokens, 8);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.name, 'embeddings');
+    assert.equal(span.attributes['gen_ai.system'], 'Cohere');
+    assert.equal(span.attributes['gen_ai.operation.name'], 'embeddings');
+    assert.equal(span.attributes['gen_ai.request.model'], 'embed-english-v3.0');
+    assert.equal(span.attributes['gen_ai.usage.input_tokens'], 8);
+    assert.equal(span.status.code, SpanStatusCode.UNSET);
+  } finally {
+    teardown();
+  }
+});
+
+test('v2 embed emits genai span with input tokens', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubV2Client();
+    await client.embed({ model: 'embed-v4.0', texts: ['hi'] });
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.attributes['gen_ai.request.model'], 'embed-v4.0');
+    assert.equal(span.attributes['gen_ai.usage.input_tokens'], 5);
+  } finally {
+    teardown();
+  }
+});
+
+test('embed records error status on failure', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    await assert.rejects(() => client.embed({ model: 'boom', texts: ['x'] }), /api error/);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.status.code, SpanStatusCode.ERROR);
+  } finally {
+    teardown();
+  }
+});
+
+test('uninstrument restores original methods', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    await client.embed({ model: 'embed-english-v3.0', texts: ['x'] });
+    assert.equal(exporter.getFinishedSpans().length, 1);
+  } finally {
+    teardown();
+  }
+});

package/tests/instrumentation/elevenlabs/full_instrumentation.test.js

@@ -0,0 +1,244 @@
+'use strict';
+
+// Tests for the bundled ElevenLabs audio instrumentor.
+//
+// The real `@elevenlabs/elevenlabs-js` clients are stubbed with methods that
+// mirror the SDK's shapes — the streaming TTS entrypoints (`convert` / `stream`)
+// resolve to a ReadableStream, `streamWithTimestamps` to an async-iterable, and
+// the unary `convertWithTimestamps` and Speech-to-Text `convert` to a response
+// object — so the instrumentor can be exercised without network access or the
+// dependency installed.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Module = require('node:module');
+
+const { trace, SpanStatusCode } = require('@opentelemetry/api');
+const {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} = require('@opentelemetry/sdk-trace-base');
+
+const {
+  ElevenLabsInstrumentor,
+} = require('../../../src/instrumentation/elevenlabs/full_instrumentation');
+
+const TTS_CLIENT_MODULE = '@elevenlabs/elevenlabs-js/api/resources/textToSpeech/client/Client';
+const STT_CLIENT_MODULE = '@elevenlabs/elevenlabs-js/api/resources/speechToText/client/Client';
+
+// The instrumentor emits via the global tracer provider. node's test runner
+// isolates each test file in its own process, so registering one provider +
+// in-memory exporter for the whole file is safe.
+const exporter = new InMemorySpanExporter();
+trace.setGlobalTracerProvider(
+  new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
+);
+
+function readableOf(chunks) {
+  let index = 0;
+  return new ReadableStream({
+    pull(controller) {
+      if (index < chunks.length) controller.enqueue(chunks[index++]);
+      else controller.close();
+    },
+  });
+}
+
+function failingReadable(error) {
+  return new ReadableStream({
+    pull() {
+      throw error;
+    },
+  });
+}
+
+function asyncIterableOf(chunks) {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const chunk of chunks) yield chunk;
+    },
+  };
+}
+
+class StubTextToSpeechClient {
+  convert(_voiceId, request = {}) {
+    if (request.text === 'boom') return Promise.resolve(failingReadable(new Error('api error')));
+    return Promise.resolve(readableOf([Buffer.from('a'), Buffer.from('b')]));
+  }
+
+  stream(_voiceId, _request = {}) {
+    return Promise.resolve(readableOf([Buffer.from('x')]));
+  }
+
+  streamWithTimestamps(_voiceId, _request = {}) {
+    return Promise.resolve(asyncIterableOf([{ audioBase64: 'eA==' }]));
+  }
+
+  convertWithTimestamps(_voiceId, _request = {}) {
+    return Promise.resolve({ audioBase64: 'YQ==' });
+  }
+}
+
+class StubSpeechToTextClient {
+  convert(request = {}) {
+    if (request.modelId === 'boom') return Promise.reject(new Error('api error'));
+    return Promise.resolve({ audioDurationSecs: 12.4, text: 'hello' });
+  }
+}
+
+async function drain(stream) {
+  const reader = stream.getReader();
+  const chunks = [];
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  return chunks;
+}
+
+async function drainIterable(iterable) {
+  const chunks = [];
+  for await (const chunk of iterable) chunks.push(chunk);
+  return chunks;
+}
+
+// Stub the elevenlabs client modules, install the instrumentor + the in-memory
+// exporter, and return a teardown that restores everything.
+function instrumented() {
+  const stubs = {
+    [TTS_CLIENT_MODULE]: { TextToSpeechClient: StubTextToSpeechClient },
+    [STT_CLIENT_MODULE]: { SpeechToTextClient: StubSpeechToTextClient },
+  };
+  const origResolve = Module._resolveFilename;
+  Module._resolveFilename = function patched(request, ...rest) {
+    if (request in stubs) return request;
+    return origResolve.call(this, request, ...rest);
+  };
+  for (const [moduleId, exports] of Object.entries(stubs)) {
+    require.cache[moduleId] = { id: moduleId, filename: moduleId, loaded: true, exports };
+  }
+
+  const instrumentor = new ElevenLabsInstrumentor();
+  instrumentor.instrument();
+
+  return () => {
+    instrumentor.uninstrument();
+    Module._resolveFilename = origResolve;
+    for (const moduleId of Object.keys(stubs)) delete require.cache[moduleId];
+    exporter.reset();
+  };
+}
+
+test('convert emits genai span with char count', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubTextToSpeechClient();
+    const stream = await client.convert('voice', { text: 'hello world', modelId: 'eleven_v2' });
+    assert.deepEqual(await drain(stream), [Buffer.from('a'), Buffer.from('b')]);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.name, 'text_to_speech');
+    assert.equal(span.attributes['gen_ai.system'], 'elevenlabs');
+    assert.equal(span.attributes['gen_ai.operation.name'], 'text_to_speech');
+    assert.equal(span.attributes['gen_ai.request.model'], 'eleven_v2');
+    assert.equal(span.attributes['gen_ai.usage.input_characters'], 'hello world'.length);
+    assert.equal(span.status.code, SpanStatusCode.UNSET);
+  } finally {
+    teardown();
+  }
+});
+
+test('convert records error status on failure', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubTextToSpeechClient();
+    const stream = await client.convert('voice', { text: 'boom', modelId: 'm' });
+    await assert.rejects(() => drain(stream), /api error/);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.status.code, SpanStatusCode.ERROR);
+  } finally {
+    teardown();
+  }
+});
+
+test('convert with timestamps emits span with char count', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubTextToSpeechClient();
+    // The unary timestamp variant resolves to a single response, not a stream.
+    const result = await client.convertWithTimestamps('voice', { text: 'hello', modelId: 'm' });
+    assert.equal(result.audioBase64, 'YQ==');
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.name, 'text_to_speech');
+    assert.equal(span.attributes['gen_ai.usage.input_characters'], 'hello'.length);
+    assert.equal(span.attributes['gen_ai.usage.input_seconds'], undefined);
+  } finally {
+    teardown();
+  }
+});
+
+test('stream with timestamps emits span with char count', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubTextToSpeechClient();
+    const stream = await client.streamWithTimestamps('voice', { text: 'hello', modelId: 'm' });
+    assert.deepEqual(await drainIterable(stream), [{ audioBase64: 'eA==' }]);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.name, 'text_to_speech');
+    assert.equal(span.attributes['gen_ai.usage.input_characters'], 'hello'.length);
+  } finally {
+    teardown();
+  }
+});
+
+test('uninstrument restores original methods', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubTextToSpeechClient();
+    assert.deepEqual(await drain(await client.stream('voice', { text: 'x' })), [Buffer.from('x')]);
+    assert.equal(exporter.getFinishedSpans().length, 1);
+  } finally {
+    teardown();
+  }
+});
+
+test('speech to text emits span with audio duration', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubSpeechToTextClient();
+    const result = await client.convert({ modelId: 'scribe_v1', file: Buffer.from('audio') });
+    assert.equal(result.audioDurationSecs, 12.4);
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.name, 'speech_to_text');
+    assert.equal(span.attributes['gen_ai.system'], 'elevenlabs');
+    assert.equal(span.attributes['gen_ai.operation.name'], 'speech_to_text');
+    assert.equal(span.attributes['gen_ai.request.model'], 'scribe_v1');
+    assert.equal(span.attributes['gen_ai.usage.input_seconds'], 12.4);
+    assert.equal(span.attributes['gen_ai.usage.input_characters'], undefined);
+    assert.equal(span.status.code, SpanStatusCode.UNSET);
+  } finally {
+    teardown();
+  }
+});
+
+test('speech to text records error status on failure', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubSpeechToTextClient();
+    await assert.rejects(
+      () => client.convert({ modelId: 'boom', file: Buffer.from('audio') }),
+      /api error/
+    );
+
+    const [span] = exporter.getFinishedSpans();
+    assert.equal(span.status.code, SpanStatusCode.ERROR);
+  } finally {
+    teardown();
+  }
+});