weflayr 0.20.6 → 0.21.0

package/src/instrumentation/openai/set_telemetry_attributes.js

@@ -0,0 +1,39 @@
+'use strict';
+
+/**
+ * OpenAI telemetry complement — the registered orchestrator.
+ *
+ * Wires every weflayr-supplied patch the traceloop OpenAI instrumentor misses:
+ *   - the audio (TTS/STT) and embeddings endpoints it doesn't cover, and
+ *   - usage it drops on the chat endpoint it does cover: streamed token usage and
+ *     non-streamed cached-prompt tokens (these enrich the traceloop instrumentor
+ *     instance, hence the `instrumentation` argument).
+ */
+
+const { OpenAIAudioInstrumentor } = require('./get_audio_telemetry');
+const { OpenAIEmbeddingsInstrumentor } = require('./get_embeddings_telemetry');
+const { OpenAIStreamingUsageInstrumentor } = require('./get_streaming_usage');
+const { OpenAIPromptCachingInstrumentor } = require('./get_cache_and_reasoning_telemetry');
+
+class OpenAITelemetryInstrumentor {
+  constructor() {
+    this._subInstrumentors = [
+      new OpenAIAudioInstrumentor(),
+      new OpenAIEmbeddingsInstrumentor(),
+      new OpenAIStreamingUsageInstrumentor(),
+      new OpenAIPromptCachingInstrumentor(),
+    ];
+  }
+
+  // The endpoint complements ignore `instrumentation`; the streaming/cache ones
+  // patch its internal handlers.
+  instrument(instrumentation) {
+    for (const sub of this._subInstrumentors) sub.instrument(instrumentation);
+  }
+
+  uninstrument() {
+    for (const sub of this._subInstrumentors) sub.uninstrument();
+  }
+}
+
+module.exports = { OpenAITelemetryInstrumentor };

package/src/otel/propagation.js

@@ -62,6 +62,11 @@ function _buildAttributes({ featureName, customerId, providerName, extraTags, sp
  * propagateMetadata call" contract.
  */
 function propagateMetadata(options, fn) {
+  const { featureName, customerId } = options ?? {};
+  if (featureName == null || customerId == null) {
+    throw new Error('propagateMetadata: `featureName` and `customerId` are required');
+  }
+
   if (typeof fn === 'function') {
     return _runWithMetadata(options, fn);
   }

package/src/otel/span-filter.js

@@ -27,18 +27,13 @@ const KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES = [
   'opentelemetry.instrumentation.google_generativeai',
   'opentelemetry.instrumentation.openai',
   'opentelemetry.instrumentation.openai_v2',
-  'opentelemetry.instrumentation.vertex_ai',
-  'opentelemetry.instrumentation.vertexai',
   'weflayr.instrumentation.elevenlabs',
   'strands-agents',
   'vllm',
   '@traceloop/instrumentation-openai',
   '@traceloop/instrumentation-anthropic',
   '@traceloop/instrumentation-bedrock',
-  '@traceloop/instrumentation-azure',
   '@traceloop/instrumentation-cohere',
-  '@traceloop/instrumentation-together',
-  '@traceloop/instrumentation-vertexai',
   '@traceloop/instrumentation-google-generativeai',
   '@traceloop/instrumentation-langchain',
   '@traceloop/instrumentation-llamaindex',

package/tests/index.test.js

@@ -14,7 +14,6 @@ const { _state } = require('../src/setup');
 const { isDefaultExportSpan } = require('../src/otel/span-filter');
 const { WeflayrSpanProcessor } = require('../src/otel/span-processor');
 const { buildExporter, _exportWithRetry } = require('../src/otel/exporter');
-const { patchOpenAIStreamingUsage } = require('../src/otel/openai-streaming-usage');
 
 const contextManager = new AsyncHooksContextManager();
 contextManager.enable();
@@ -478,12 +477,48 @@ test('propagateMetadata providerName writes attribute', async () => {
   await provider.shutdown();
 });
 
+test('propagateMetadata requires featureName and customerId', () => {
+  assert.throws(() => weflayr.propagateMetadata({}), /featureName.*customerId.*required/);
+  assert.throws(() => weflayr.propagateMetadata({ featureName: 'f' }), /required/);
+  assert.throws(() => weflayr.propagateMetadata({ customerId: 'c' }), /required/);
+});
+
 // ── autoInstrument ────────────────────────────────────────────────────────────
 
 function _ensureSetupDone() {
   if (!_state.processor) weflayr.setup({ apiKey: 'test-key' });
 }
 
+// Instrumenting OPENAI also runs the bundled audio + embeddings instrumentors,
+// which eagerly require the openai resource modules. Stub them so autoInstrument
+// tests don't need the real `openai` dependency installed.
+const OPENAI_RESOURCE_STUBS = {
+  'openai/resources/audio/speech': {
+    Speech: class {
+      create() {}
+    },
+  },
+  'openai/resources/audio/transcriptions': {
+    Transcriptions: class {
+      create() {}
+    },
+  },
+  'openai/resources/embeddings': {
+    Embeddings: class {
+      create() {}
+    },
+  },
+};
+
+function _stubOpenAIResources() {
+  for (const [id, exports] of Object.entries(OPENAI_RESOURCE_STUBS)) {
+    require.cache[id] = { id, filename: id, loaded: true, exports };
+  }
+  return () => {
+    for (const id of Object.keys(OPENAI_RESOURCE_STUBS)) delete require.cache[id];
+  };
+}
+
 test('autoInstrument throws "install package X" when the dep is missing', async () => {
   _ensureSetupDone();
   await assert.rejects(
@@ -517,7 +552,7 @@ test('autoInstrument loads a stubbed Traceloop package', async () => {
   }
   const origResolve = Module._resolveFilename;
   Module._resolveFilename = function patched(req, ...rest) {
-    if (req === '@traceloop/instrumentation-openai') return req;
+    if (req === '@traceloop/instrumentation-openai' || req in OPENAI_RESOURCE_STUBS) return req;
     return origResolve.call(this, req, ...rest);
   };
   require.cache['@traceloop/instrumentation-openai'] = {
@@ -526,12 +561,14 @@ test('autoInstrument loads a stubbed Traceloop package', async () => {
     loaded: true,
     exports: { OpenAIInstrumentation: FakeOpenAIInstrumentation },
   };
+  const cleanupOpenAI = _stubOpenAIResources();
   try {
     await weflayr.autoInstrument({ aiSdks: weflayr.AiSdk.OPENAI });
     assert.equal(constructed, true);
   } finally {
     Module._resolveFilename = origResolve;
     delete require.cache['@traceloop/instrumentation-openai'];
+    cleanupOpenAI();
   }
 });
 
@@ -564,7 +601,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
   }
   const origResolve = Module._resolveFilename;
   Module._resolveFilename = function patched(req, ...rest) {
-    if (req === '@traceloop/instrumentation-openai') return req;
+    if (req === '@traceloop/instrumentation-openai' || req in OPENAI_RESOURCE_STUBS) return req;
     return origResolve.call(this, req, ...rest);
   };
   require.cache['@traceloop/instrumentation-openai'] = {
@@ -573,6 +610,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
     loaded: true,
     exports: { OpenAIInstrumentation: FakeOpenAIInstrumentation },
   };
+  const cleanupOpenAI = _stubOpenAIResources();
   try {
     await weflayr.autoInstrument({
       aiSdks: weflayr.AiSdk.OPENAI,
@@ -582,6 +620,7 @@ test('autoInstrument runs setup when apiKey passed', async () => {
   } finally {
     Module._resolveFilename = origResolve;
     delete require.cache['@traceloop/instrumentation-openai'];
+    cleanupOpenAI();
   }
 });
 
@@ -641,47 +680,6 @@ test('exporter gives up after maxRetries on persistent 503', async () => {
   assert.equal(calls, 3);
 });
 
-// ── OpenAI streaming usage backfill ─────────────────────────────────────────────
-
-// Build a fake OpenAI instrumentor whose streaming handler mimics traceloop's:
-// it yields content chunks, then a final usage chunk (as OpenAI sends when
-// `stream_options.include_usage` is set) without recording usage itself. The
-// patch should read the usage chunk and set the token attributes on the span.
-function makeFakeOpenAIInstrumentation() {
-  return {
-    _streamingWrapPromise() {
-      return (async function* originalStream() {
-        yield { id: 'chatcmpl-1', model: 'gpt-4o-mini', choices: [{ delta: { content: 'Hi' } }] };
-        yield {
-          id: 'chatcmpl-1',
-          model: 'gpt-4o-mini',
-          choices: [],
-          usage: { prompt_tokens: 13, completion_tokens: 2, total_tokens: 15 },
-        };
-      })();
-    },
-  };
-}
-
-test('patchOpenAIStreamingUsage sets token attributes from the streamed usage chunk', async () => {
-  const span = makeFakeSpan();
-  const instrumentation = makeFakeOpenAIInstrumentation();
-  patchOpenAIStreamingUsage(instrumentation);
-
-  const stream = instrumentation._streamingWrapPromise({ span });
-  for await (const chunk of stream) void chunk;
-
-  assert.equal(span.attributes['gen_ai.usage.input_tokens'], 13);
-  assert.equal(span.attributes['gen_ai.usage.output_tokens'], 2);
-  assert.equal(span.attributes['gen_ai.usage.total_tokens'], 15);
-});
-
-test('patchOpenAIStreamingUsage is a no-op when the streaming handler is absent', () => {
-  const instrumentation = {};
-  patchOpenAIStreamingUsage(instrumentation);
-  assert.equal(instrumentation._streamingWrapPromise, undefined);
-});
-
 // ── flush ─────────────────────────────────────────────────────────────────────
 
 test('flush does not throw', async () => {

package/tests/instrumentation/anthropic/get_cache_and_reasoning_telemetry.test.js

@@ -0,0 +1,125 @@
+'use strict';
+
+// Tests for the Anthropic cached-prompt token complement. It wraps the primary
+// instrumentor's `_endSpan({ span, type, result })` to stamp
+// gen_ai.usage.cache_read_input_tokens / cache_creation_input_tokens from the
+// response's `usage.cache_read_input_tokens` / `cache_creation_input_tokens`.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const {
+  AnthropicPromptCachingInstrumentor,
+} = require('../../../src/instrumentation/anthropic/get_cache_and_reasoning_telemetry');
+
+function fakeSpan() {
+  const attrs = {};
+  return {
+    attrs,
+    isRecording: () => true,
+    setAttribute(key, value) {
+      attrs[key] = value;
+    },
+  };
+}
+
+test('stamps cache write and still calls the original _endSpan', () => {
+  const calls = [];
+  const instrumentation = {
+    _endSpan(args) {
+      calls.push(args);
+    },
+  };
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._endSpan({
+    span,
+    type: 'completion',
+    result: {
+      usage: { input_tokens: 10, cache_creation_input_tokens: 5269, cache_read_input_tokens: 0 },
+    },
+  });
+
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], 5269);
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(calls.length, 1);
+});
+
+test('stamps cache read on a warm call', () => {
+  const instrumentation = { _endSpan() {} };
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._endSpan({
+    span,
+    type: 'completion',
+    result: {
+      usage: { input_tokens: 10, cache_creation_input_tokens: 0, cache_read_input_tokens: 5271 },
+    },
+  });
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 5271);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+});
+
+test('sets nothing for an uncached call', () => {
+  const instrumentation = { _endSpan() {} };
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._endSpan({ span, type: 'completion', result: { usage: { input_tokens: 13 } } });
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+});
+
+test('stamps extended thinking tokens from output_tokens_details', () => {
+  const instrumentation = { _endSpan() {} };
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._endSpan({
+    span,
+    type: 'completion',
+    result: { usage: { input_tokens: 10, output_tokens_details: { thinking_tokens: 150 } } },
+  });
+
+  assert.equal(span.attrs['gen_ai.usage.output_tokens_details.thinking_tokens'], 150);
+});
+
+test('stamps cache + raw input from the streamed message_start usage', async () => {
+  const instrumentation = {
+    _endSpan() {},
+    _streamingWrapPromise() {
+      return (async function* () {
+        yield {
+          type: 'message_start',
+          message: {
+            usage: {
+              input_tokens: 10,
+              cache_read_input_tokens: 5270,
+              cache_creation_input_tokens: 0,
+            },
+          },
+        };
+        yield { type: 'content_block_delta', delta: { text: 'OK' } };
+      })();
+    },
+  };
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  for await (const _chunk of instrumentation._streamingWrapPromise('client', 'mod', { span })) {
+    void _chunk;
+  }
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 5270);
+  assert.equal(span.attrs['gen_ai.usage.input_tokens'], 10);
+});
+
+test('is a no-op when the instrumentor has no _endSpan', () => {
+  const instrumentation = {};
+  new AnthropicPromptCachingInstrumentor().instrument(instrumentation);
+  assert.equal(instrumentation._endSpan, undefined);
+});

package/tests/instrumentation/bedrock/get_cache_telemetry.test.js

@@ -0,0 +1,102 @@
+'use strict';
+
+// Tests for the Bedrock cached-prompt token complement. It wraps the primary
+// instrumentor's `_wrapPromise(span, promise, modelId)` to read Nova's cache-token
+// counts off the decoded InvokeModel response body and stamp them as
+// gen_ai.usage.cache_* attributes.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+
+const {
+  BedrockPromptCachingInstrumentor,
+} = require('../../../src/instrumentation/bedrock/get_cache_telemetry');
+
+function fakeSpan() {
+  const attrs = {};
+  return {
+    attrs,
+    isRecording: () => true,
+    setAttribute(key, value) {
+      attrs[key] = value;
+    },
+  };
+}
+
+function encode(obj) {
+  return new TextEncoder().encode(JSON.stringify(obj));
+}
+
+// Pass-through original: resolve to whatever promise it was handed.
+function passthroughInstrumentation() {
+  return { _wrapPromise: (_span, promise) => Promise.resolve(promise) };
+}
+
+test('stamps cache read/write from a buffered InvokeModel body', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  const body = encode({
+    usage: { cacheReadInputTokenCount: 100, cacheWriteInputTokenCount: 1300 },
+  });
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'amazon.nova-micro-v1:0');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 100);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], 1300);
+});
+
+test('sets nothing when cache counts are zero', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  const body = encode({ usage: { cacheReadInputTokenCount: 0, cacheWriteInputTokenCount: 0 } });
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'm');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+});
+
+test('skips a streamed (async-iterable) body without consuming it', async () => {
+  const instrumentation = passthroughInstrumentation();
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  let consumed = false;
+  const body = (async function* () {
+    consumed = true;
+    yield {};
+  })();
+  await instrumentation._wrapPromise(span, Promise.resolve({ body }), 'm');
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], undefined);
+  assert.equal(consumed, false);
+});
+
+test('stamps cache from the streamed metadata chunk and calls the original', () => {
+  const calls = [];
+  const instrumentation = {
+    _handleNovaStreamingMetadata(span, parsed) {
+      calls.push(parsed);
+    },
+  };
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+
+  const span = fakeSpan();
+  instrumentation._handleNovaStreamingMetadata(span, {
+    metadata: {
+      usage: { inputTokens: 3, cacheReadInputTokenCount: 1458, cacheWriteInputTokenCount: 0 },
+    },
+  });
+
+  assert.equal(span.attrs['gen_ai.usage.cache_read_input_tokens'], 1458);
+  assert.equal(span.attrs['gen_ai.usage.cache_creation_input_tokens'], undefined);
+  assert.equal(calls.length, 1);
+});
+
+test('is a no-op when the instrumentor has no _wrapPromise', () => {
+  const instrumentation = {};
+  new BedrockPromptCachingInstrumentor().instrument(instrumentation);
+  assert.equal(instrumentation._wrapPromise, undefined);
+});

package/tests/instrumentation/cohere/get_chat_telemetry.test.js

@@ -0,0 +1,148 @@
+'use strict';
+
+// Tests for the Cohere chat token-usage enrichment complement.
+//
+// The cohere-ai v1 `CohereClient` is stubbed with an internal `__chat` method
+// that mirrors the SDK shape — resolving to `{ data: { meta: { billedUnits } } }`
+// — so the complement can be exercised without network access or the dependency
+// installed. The complement enriches whichever span is active when `__chat` runs
+// (in production, the span traceloop opens around the public `chat`), so the
+// tests start a span and make it active before invoking `__chat`.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const Module = require('node:module');
+
+const { trace, context } = require('@opentelemetry/api');
+const {
+  BasicTracerProvider,
+  InMemorySpanExporter,
+  SimpleSpanProcessor,
+} = require('@opentelemetry/sdk-trace-base');
+const { AsyncLocalStorageContextManager } = require('@opentelemetry/context-async-hooks');
+
+const {
+  CohereChatTokenUsageInstrumentor,
+} = require('../../../src/instrumentation/cohere/get_chat_telemetry');
+
+const CLIENT_MODULE = 'cohere-ai/Client';
+
+const exporter = new InMemorySpanExporter();
+trace.setGlobalTracerProvider(
+  new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] })
+);
+context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable());
+const tracer = trace.getTracer('test');
+
+class StubCohereClient {
+  __chat(request = {}) {
+    if (request.model === 'boom') return Promise.reject(new Error('api error'));
+    return Promise.resolve({
+      data: { text: 'hi', meta: { billedUnits: { inputTokens: 7, outputTokens: 3 } } },
+    });
+  }
+}
+
+function instrumented() {
+  const stubs = { [CLIENT_MODULE]: { CohereClient: StubCohereClient } };
+  const origResolve = Module._resolveFilename;
+  Module._resolveFilename = function patched(request, ...rest) {
+    if (request in stubs) return request;
+    return origResolve.call(this, request, ...rest);
+  };
+  for (const [moduleId, exports] of Object.entries(stubs)) {
+    require.cache[moduleId] = { id: moduleId, filename: moduleId, loaded: true, exports };
+  }
+
+  const instrumentor = new CohereChatTokenUsageInstrumentor();
+  instrumentor.instrument();
+
+  return () => {
+    instrumentor.uninstrument();
+    Module._resolveFilename = origResolve;
+    for (const moduleId of Object.keys(stubs)) delete require.cache[moduleId];
+    exporter.reset();
+  };
+}
+
+// Runs `__chat` inside an active span, mirroring the context traceloop sets up
+// around the public `chat`, and returns the finished span.
+async function callChatWithinSpan(client, request) {
+  const span = tracer.startSpan('cohere.chat');
+  await context.with(trace.setSpan(context.active(), span), () => client.__chat(request));
+  span.end();
+  return exporter.getFinishedSpans().find((finished) => finished.name === 'cohere.chat');
+}
+
+test('enriches the active chat span with input and output tokens', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    const span = await callChatWithinSpan(client, { model: 'command-r-08-2024', message: 'hi' });
+
+    assert.equal(span.attributes['gen_ai.usage.input_tokens'], 7);
+    assert.equal(span.attributes['gen_ai.usage.output_tokens'], 3);
+  } finally {
+    teardown();
+  }
+});
+
+test('leaves the response unchanged', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    let result;
+    const span = tracer.startSpan('cohere.chat');
+    await context.with(trace.setSpan(context.active(), span), async () => {
+      result = await client.__chat({ model: 'command-r-08-2024', message: 'hi' });
+    });
+    span.end();
+
+    assert.equal(result.data.text, 'hi');
+    assert.equal(result.data.meta.billedUnits.inputTokens, 7);
+  } finally {
+    teardown();
+  }
+});
+
+test('does nothing when no span is active', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    // No active span — must not throw, just pass the response through.
+    const result = await client.__chat({ model: 'command-r-08-2024', message: 'hi' });
+    assert.equal(result.data.meta.billedUnits.outputTokens, 3);
+    assert.equal(exporter.getFinishedSpans().length, 0);
+  } finally {
+    teardown();
+  }
+});
+
+test('propagates errors without enriching the span', async () => {
+  const teardown = instrumented();
+  try {
+    const client = new StubCohereClient();
+    const span = tracer.startSpan('cohere.chat');
+    await assert.rejects(
+      () =>
+        context.with(trace.setSpan(context.active(), span), () =>
+          client.__chat({ model: 'boom', message: 'x' })
+        ),
+      /api error/
+    );
+    span.end();
+
+    const finished = exporter.getFinishedSpans().find((each) => each.name === 'cohere.chat');
+    assert.equal(finished.attributes['gen_ai.usage.input_tokens'], undefined);
+  } finally {
+    teardown();
+  }
+});
+
+test('uninstrument restores the original method', async () => {
+  const teardown = instrumented();
+  const { CohereClient } = require(CLIENT_MODULE);
+  const wrapped = CohereClient.prototype.__chat;
+  teardown();
+  assert.notEqual(CohereClient.prototype.__chat, wrapped);
+});