toad-eye 2.2.0 → 2.3.0

package/dist/core/spans.d.ts

@@ -1,7 +1,7 @@
-import type { LLMSpanAttributes } from "../types/index.js";
+import type { LLMProvider } from "../types/index.js";
 /** Input for traceLLMCall — what the user knows before calling the LLM */
 export interface LLMCallInput {
-    readonly provider: LLMSpanAttributes["provider"];
+    readonly provider: LLMProvider;
     readonly model: string;
     readonly prompt: string;
     readonly temperature?: number | undefined;
@@ -13,6 +13,8 @@ export interface LLMCallOutput {
     readonly completion: string;
     readonly inputTokens: number;
     readonly outputTokens: number;
-    readonly cost: number;
+    /** Cost in USD. If omitted, auto-calculated from model pricing table. */
+    readonly cost?: number | undefined;
 }
+export declare function processContent(text: string): string | undefined;
 export declare function traceLLMCall(input: LLMCallInput, fn: () => Promise<LLMCallOutput>): Promise<LLMCallOutput>;

package/dist/core/spans.js

@@ -1,32 +1,73 @@
 import { createHash } from "node:crypto";
-import { trace, diag, SpanStatusCode } from "@opentelemetry/api";
+import { trace, SpanStatusCode } from "@opentelemetry/api";
 import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "../types/index.js";
-import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordBudgetExceeded, recordBudgetBlocked, recordBudgetDowngraded, } from "./metrics.js";
+import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordBudgetExceeded, recordBudgetBlocked, recordBudgetDowngraded, recordResponseEmpty, recordResponseLatencyPerToken, } from "./metrics.js";
 import { getConfig, getBudgetTracker } from "./tracer.js";
-import { GEN_AI_ATTRS as ATTRS } from "../types/index.js";
+import { calculateCost } from "./pricing.js";
+import { ToadBudgetExceededError } from "../budget/index.js";
 const tracer = trace.getTracer(INSTRUMENTATION_NAME);
 let saltWarningEmitted = false;
+// Track the last config ref so saltWarningEmitted resets across re-inits without circular imports
+let lastConfigRef = null;
 function sha256(text) {
     const config = getConfig();
     const salt = config?.salt ?? "";
+    // Auto-reset saltWarningEmitted when SDK is re-initialized (config ref changes)
+    if (config !== lastConfigRef) {
+        lastConfigRef = config;
+        saltWarningEmitted = false;
+    }
     if (config?.hashContent && !config.salt && !saltWarningEmitted) {
-        diag.warn("toad-eye: hashContent is enabled without salt — short strings may be reversible. Set salt in config for stronger privacy.");
+        console.warn("toad-eye: hashContent is enabled without salt — short strings may be reversible. Set salt in config for stronger privacy.");
         saltWarningEmitted = true;
     }
     return createHash("sha256")
         .update(salt + text)
         .digest("hex");
 }
-function processContent(text) {
+// Built-in PII patterns — enabled via redactDefaults: true
+// Note: no `g` flag here — a fresh RegExp is created per call to avoid lastIndex state issues
+const DEFAULT_REDACT_PATTERNS = [
+    /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/, // email
+    /\b\d{3}-\d{2}-\d{4}\b/, // SSN (US)
+    /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/, // credit card
+    /\b\+?\d{1,3}[\s.-]?\(?\d{2,4}\)?[\s.-]?\d{3,4}[\s.-]?\d{3,4}\b/, // phone
+];
+function applyRedaction(text, pattern) {
+    let count = 0;
+    const regex = new RegExp(pattern.source, "g");
+    const result = text.replace(regex, () => {
+        count++;
+        return "[REDACTED]";
+    });
+    return [result, count];
+}
+export function processContent(text) {
     const config = getConfig();
     if (config?.recordContent === false)
         return undefined;
     let processed = text;
+    let totalRedacted = 0;
+    // Apply default PII patterns if enabled
+    if (config?.redactDefaults) {
+        for (const pattern of DEFAULT_REDACT_PATTERNS) {
+            const [result, count] = applyRedaction(processed, pattern);
+            processed = result;
+            totalRedacted += count;
+        }
+    }
+    // Apply custom patterns
     if (config?.redactPatterns?.length) {
         for (const pattern of config.redactPatterns) {
-            processed = processed.replace(pattern, "[REDACTED]");
+            const [result, count] = applyRedaction(processed, pattern);
+            processed = result;
+            totalRedacted += count;
         }
     }
+    // Audit mode — log a summary only, never the original content (would re-expose PII)
+    if (config?.auditMasking && totalRedacted > 0) {
+        console.log(`[toad-eye audit] Content masked: ${totalRedacted} pattern(s) applied, ${text.length} chars → ${processed.length} chars`);
+    }
     if (config?.hashContent) {
         return `sha256:${sha256(processed)}`;
     }
@@ -62,14 +103,14 @@ function recordBaseMetrics(duration, provider, model, attrs) {
     recordRequest(provider, model, attrs);
     recordRequestDuration(duration, provider, model, attrs);
 }
-function setSuccessAttributes(span, input, output) {
+function setSuccessAttributes(span, input, output, resolvedCost) {
     const completion = processContent(output.completion);
     span.setAttributes({
         ...(completion !== undefined && { [GEN_AI_ATTRS.COMPLETION]: completion }),
         [GEN_AI_ATTRS.RESPONSE_MODEL]: input.model,
         [GEN_AI_ATTRS.INPUT_TOKENS]: output.inputTokens,
         [GEN_AI_ATTRS.OUTPUT_TOKENS]: output.outputTokens,
-        [GEN_AI_ATTRS.COST]: output.cost,
+        [GEN_AI_ATTRS.COST]: resolvedCost,
         [GEN_AI_ATTRS.STATUS]: "success",
         [GEN_AI_ATTRS.FINISH_REASONS]: ["stop"],
     });
@@ -90,12 +131,18 @@ function setErrorAttributes(span, message) {
     });
 }
 export async function traceLLMCall(input, fn) {
+    if (!getConfig()) {
+        console.warn("toad-eye: traceLLMCall called before initObservability() — no telemetry will be recorded.");
+    }
     const budget = getBudgetTracker();
     let effectiveInput = input;
+    // Estimated cost for reservation — reduces race window for concurrent requests.
+    // Uses conservative token estimates; actual cost is reconciled in recordCost.
+    const estimatedCost = budget ? calculateCost(input.model, 500, 200) : 0;
     // Budget check BEFORE the LLM call
     if (budget) {
-        const userId = input.attributes?.[ATTRS.USER_ID];
-        const override = budget.checkBefore(input.provider, input.model, userId);
+        const userId = input.attributes?.[GEN_AI_ATTRS.USER_ID];
+        const override = budget.checkBefore(input.provider, input.model, userId, estimatedCost);
         if (override) {
             // Downgrade mode — use modified provider/model
             effectiveInput = {
@@ -103,7 +150,7 @@ export async function traceLLMCall(input, fn) {
                 provider: override.provider,
                 model: override.model,
             };
-            recordBudgetDowngraded("daily");
+            recordBudgetDowngraded(override.budget);
         }
     }
     return tracer.startActiveSpan(`gen_ai.${effectiveInput.provider}.${effectiveInput.model}`, async (span) => {
@@ -113,14 +160,23 @@ export async function traceLLMCall(input, fn) {
             const output = await fn();
             const duration = performance.now() - start;
             const attrs = resolveAttributes(effectiveInput);
-            setSuccessAttributes(span, effectiveInput, output);
+            const resolvedCost = output.cost ??
+                calculateCost(effectiveInput.model, output.inputTokens, output.outputTokens);
+            setSuccessAttributes(span, effectiveInput, output, resolvedCost);
             recordBaseMetrics(duration, effectiveInput.provider, effectiveInput.model, attrs);
-            recordRequestCost(output.cost, effectiveInput.provider, effectiveInput.model, attrs);
+            recordRequestCost(resolvedCost, effectiveInput.provider, effectiveInput.model, attrs);
             recordTokens(output.inputTokens + output.outputTokens, effectiveInput.provider, effectiveInput.model, attrs);
-            // Budget recording AFTER the call
+            // Proxy quality metrics — no extra dependencies, response text analysis only
+            if (output.completion.trim() === "") {
+                recordResponseEmpty(effectiveInput.provider, effectiveInput.model, attrs);
+            }
+            if (output.outputTokens > 0) {
+                recordResponseLatencyPerToken(duration / output.outputTokens, effectiveInput.provider, effectiveInput.model, attrs);
+            }
+            // Budget recording AFTER the call — releases the cost reservation
             if (budget) {
-                const userId = effectiveInput.attributes?.[ATTRS.USER_ID];
-                const exceeded = budget.recordCost(output.cost, effectiveInput.model, userId);
+                const userId = effectiveInput.attributes?.[GEN_AI_ATTRS.USER_ID];
+                const exceeded = budget.recordCost(resolvedCost, effectiveInput.model, userId, estimatedCost);
                 if (exceeded) {
                     recordBudgetExceeded(exceeded.budget);
                     console.warn(`toad-eye: ${exceeded.budget} budget exceeded — limit $${exceeded.limit}, current $${exceeded.current.toFixed(2)}`);
@@ -135,10 +191,13 @@ export async function traceLLMCall(input, fn) {
             setErrorAttributes(span, message);
             recordBaseMetrics(duration, effectiveInput.provider, effectiveInput.model, attrs);
             recordError(effectiveInput.provider, effectiveInput.model, attrs);
-            // If this was a budget block, record the metric
-            if (error instanceof Error &&
-                error.name === "ToadBudgetExceededError") {
-                recordBudgetBlocked("daily");
+            // Release cost reservation on any error (no cost was incurred)
+            if (budget) {
+                budget.releaseReservation(estimatedCost);
+            }
+            // If this was a budget block, record the metric with the correct budget type
+            if (error instanceof ToadBudgetExceededError) {
+                recordBudgetBlocked(error.budget);
             }
             throw error;
         }

package/dist/core/tracer.d.ts

@@ -1,8 +1,5 @@
 import type { ToadEyeConfig } from "../types/index.js";
 import { BudgetTracker } from "../budget/index.js";
-import "../instrumentations/openai.js";
-import "../instrumentations/anthropic.js";
-import "../instrumentations/gemini.js";
 export declare function getConfig(): ToadEyeConfig | null;
 export declare function getBudgetTracker(): BudgetTracker | null;
 export declare function initObservability(config: ToadEyeConfig): void;

package/dist/core/tracer.js

@@ -3,14 +3,13 @@ import { resourceFromAttributes } from "@opentelemetry/resources";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
 import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { TraceIdRatioBasedSampler, ParentBasedSampler, } from "@opentelemetry/sdk-trace-node";
 import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-import { initMetrics } from "./metrics.js";
+import { initMetrics, resetMetrics } from "./metrics.js";
+import { resetCustomPricing } from "./pricing.js";
 import { enableAll, disableAll } from "../instrumentations/registry.js";
 import { BudgetTracker } from "../budget/index.js";
-// Side-effect imports: register provider instrumentations
-import "../instrumentations/openai.js";
-import "../instrumentations/anthropic.js";
-import "../instrumentations/gemini.js";
+import { ToadEyeAISpanProcessor } from "../vercel.js";
 const DEFAULT_ENDPOINT = "http://localhost:4318";
 const DEFAULT_CLOUD_ENDPOINT = "https://cloud.toad-eye.dev";
 let sdk = null;
@@ -45,8 +44,10 @@ function resolveTransport(config) {
     return { endpoint, headers, isCloudMode };
 }
 export function initObservability(config) {
-    if (sdk)
+    if (sdk) {
+        console.warn("toad-eye: initObservability() already called. Call shutdown() first to reconfigure.");
         return;
+    }
     validateConfig(config);
     const { endpoint, headers, isCloudMode } = resolveTransport(config);
     const resource = resourceFromAttributes({
@@ -64,14 +65,38 @@ export function initObservability(config) {
         exporter: metricExporter,
         exportIntervalMillis: isCloudMode ? 10_000 : 5_000,
     });
+    // Add Vercel AI SDK SpanProcessor if 'ai' is in instrument list
+    const spanProcessors = config.instrument?.includes("ai")
+        ? [new ToadEyeAISpanProcessor()]
+        : [];
+    // SDK-side head sampling (default: 1.0 = send everything to Collector)
+    const sdkRate = config.sampling?.sdkRate ?? 1.0;
+    const sampler = sdkRate < 1.0
+        ? new ParentBasedSampler({
+            root: new TraceIdRatioBasedSampler(sdkRate),
+        })
+        : undefined;
     sdk = new NodeSDK({
         resource,
         traceExporter,
         metricReader,
+        spanProcessors,
+        ...(sampler !== undefined && { sampler }),
     });
     sdk.start();
     currentConfig = config;
     initMetrics();
+    // Non-blocking connectivity check — warns the user if the OTel Collector is unreachable.
+    // Fire-and-forget: does not block initObservability(). Cloud mode skips this check.
+    if (!isCloudMode) {
+        void fetch(`${endpoint}/v1/traces`, {
+            method: "POST",
+            body: "[]",
+            signal: AbortSignal.timeout(2000),
+        }).catch(() => {
+            console.warn(`toad-eye: cannot reach OTel Collector at ${endpoint} — no telemetry will be exported. Is the stack running? Run: npx toad-eye up`);
+        });
+    }
     if (isCloudMode) {
         const masked = config.apiKey
             ? config.apiKey.slice(0, 8) + "..." + config.apiKey.slice(-4)
@@ -82,7 +107,13 @@ export function initObservability(config) {
         budgetTracker = new BudgetTracker(config.budgets, config.onBudgetExceeded ?? "warn", config.downgradeCallback);
     }
     if (config.instrument?.length) {
-        enableAll(config.instrument);
+        // Filter out 'ai' — it uses SpanProcessor, not monkey-patching
+        const patchProviders = config.instrument.filter((i) => i !== "ai");
+        if (patchProviders.length > 0) {
+            // enableAll is async (lazy-loads provider modules on first call).
+            // Fire-and-forget: patching completes before any async SDK call.
+            void enableAll(patchProviders);
+        }
     }
 }
 export async function shutdown() {
@@ -93,4 +124,6 @@ export async function shutdown() {
     sdk = null;
     currentConfig = null;
     budgetTracker = null;
+    resetMetrics();
+    resetCustomPricing();
 }

package/dist/drift/baseline.js

@@ -12,5 +12,11 @@ export function loadBaseline(filepath) {
     if (!existsSync(filepath))
         return undefined;
     const raw = readFileSync(filepath, "utf-8");
-    return JSON.parse(raw);
+    const parsed = JSON.parse(raw);
+    if (typeof parsed !== "object" ||
+        parsed === null ||
+        !Array.isArray(parsed["embeddings"])) {
+        throw new Error(`toad-eye: baseline file at "${filepath}" is malformed — missing "embeddings" array`);
+    }
+    return parsed;
 }

package/dist/drift/monitor.js

@@ -39,13 +39,13 @@ export function createDriftMonitor(config) {
     return {
         check: performCheck,
         checkInBackground(response, providerName, model) {
-            const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("Drift check timeout (5s)")), 5000));
-            void Promise.race([
-                performCheck(response, providerName, model),
-                timeout,
-            ]).catch((err) => {
+            let timer;
+            const timeout = new Promise((_, reject) => (timer = setTimeout(() => reject(new Error("Drift check timeout (5s)")), 5000)));
+            void Promise.race([performCheck(response, providerName, model), timeout])
+                .catch((err) => {
                 console.warn(`[toad-eye] Drift check failed (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
-            });
+            })
+                .finally(() => clearTimeout(timer));
         },
     };
 }

package/dist/index.d.ts

@@ -13,5 +13,10 @@ export type { ExportTraceOptions } from "./export.js";
 export { calculateCost, setCustomPricing, getModelPricing, } from "./core/pricing.js";
 export type { ModelPricing } from "./core/pricing.js";
 export type { LLMCallInput, LLMCallOutput } from "./core/spans.js";
-export type { ToadEyeConfig, LLMSpanAttributes, LLMProvider, SpanStatus, MetricName, AgentStepType, AgentStepInput, GuardMode, GuardResult, } from "./types/index.js";
-export { GEN_AI_METRICS, GEN_AI_ATTRS, LLM_METRICS, LLM_ATTRS, } from "./types/index.js";
+export type { ToadEyeConfig, LLMSpanAttributes, LLMProvider, SpanStatus, MetricName, AgentStepType, AgentStepInput, AgentQueryOptions, GuardMode, GuardResult, InstrumentTarget, SamplingConfig, } from "./types/index.js";
+export { GEN_AI_METRICS, GEN_AI_ATTRS } from "./types/index.js";
+/** @deprecated Use GEN_AI_METRICS instead */
+export { LLM_METRICS } from "./types/index.js";
+/** @deprecated Use GEN_AI_ATTRS instead */
+export { LLM_ATTRS } from "./types/index.js";
+export { ToadEyeAISpanProcessor, withToadEye } from "./vercel.js";

package/dist/index.js

@@ -7,4 +7,9 @@ export { recordGuardResult } from "./guard.js";
 export { createDriftMonitor, createOpenAIEmbeddingProvider, cosineSimilarity, cosineDrift, saveBaseline, loadBaseline, } from "./drift/index.js";
 export { exportTrace, fetchTrace, traceToEvalYaml } from "./export.js";
 export { calculateCost, setCustomPricing, getModelPricing, } from "./core/pricing.js";
-export { GEN_AI_METRICS, GEN_AI_ATTRS, LLM_METRICS, LLM_ATTRS, } from "./types/index.js";
+export { GEN_AI_METRICS, GEN_AI_ATTRS } from "./types/index.js";
+/** @deprecated Use GEN_AI_METRICS instead */
+export { LLM_METRICS } from "./types/index.js";
+/** @deprecated Use GEN_AI_ATTRS instead */
+export { LLM_ATTRS } from "./types/index.js";
+export { ToadEyeAISpanProcessor, withToadEye } from "./vercel.js";

package/dist/instrumentations/anthropic.d.ts

@@ -1 +1 @@
-export {};
+export declare const anthropicInstrumentation: import("./types.js").Instrumentation;

package/dist/instrumentations/anthropic.js

@@ -1,4 +1,3 @@
-import { register } from "./registry.js";
 import { createInstrumentation } from "./create.js";
 function extractMessages(messages) {
     if (!Array.isArray(messages))
@@ -59,8 +58,8 @@ const messagesCreate = {
         }
     },
 };
-register(createInstrumentation({
+export const anthropicInstrumentation = createInstrumentation({
     name: "anthropic",
     moduleName: "@anthropic-ai/sdk",
     patches: [messagesCreate],
-}));
+});

package/dist/instrumentations/create.js

@@ -1,8 +1,10 @@
 import { createRequire } from "node:module";
 import { trace, context, diag, SpanStatusCode, } from "@opentelemetry/api";
-import { traceLLMCall } from "../core/spans.js";
+import { traceLLMCall, processContent } from "../core/spans.js";
 import { calculateCost } from "../core/pricing.js";
-import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordTimeToFirstToken, } from "../core/metrics.js";
+import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordTimeToFirstToken, recordResponseEmpty, recordResponseLatencyPerToken, recordBudgetExceeded, recordBudgetDowngraded, } from "../core/metrics.js";
+import { getConfig, getBudgetTracker } from "../core/tracer.js";
+import { ToadBudgetExceededError } from "../budget/index.js";
 import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "../types/index.js";
 const require = createRequire(import.meta.url);
 function isModuleInstalled(moduleName) {
@@ -64,17 +66,55 @@ function createStreamingHandler(
 original, patch, providerName) {
     const tracer = trace.getTracer(INSTRUMENTATION_NAME);
     return async function handleStreaming(thisArg, body, rest) {
-        const req = patch.extractRequest(body);
+        // Pass thisArg so extractRequest can access instance properties (e.g., Gemini model name)
+        const req = patch.extractRequest(body, thisArg);
         const start = performance.now();
-        const response = await original.call(thisArg, body, ...rest);
-        const span = tracer.startSpan(`gen_ai.${providerName}.${req.model}`);
+        // Budget check BEFORE the LLM call — mirrors traceLLMCall behavior
+        const budget = getBudgetTracker();
+        const config = getConfig();
+        const userId = config?.attributes?.[GEN_AI_ATTRS.USER_ID];
+        const estimatedCost = budget ? calculateCost(req.model, 500, 200) : 0;
+        let effectiveProvider = providerName;
+        let effectiveModel = req.model;
+        if (budget) {
+            const override = budget.checkBefore(providerName, req.model, userId, estimatedCost);
+            if (override) {
+                effectiveProvider = override.provider;
+                effectiveModel = override.model;
+                recordBudgetDowngraded(override.budget);
+            }
+        }
+        const span = tracer.startSpan(`gen_ai.${effectiveProvider}.${effectiveModel}`);
         const ctx = trace.setSpan(context.active(), span);
+        const sessionId = config?.sessionExtractor?.() ?? config?.sessionId;
         span.setAttributes({
-            [GEN_AI_ATTRS.PROVIDER]: providerName,
-            [GEN_AI_ATTRS.REQUEST_MODEL]: req.model,
+            [GEN_AI_ATTRS.PROVIDER]: effectiveProvider,
+            [GEN_AI_ATTRS.REQUEST_MODEL]: effectiveModel,
             [GEN_AI_ATTRS.TEMPERATURE]: req.temperature ?? 1.0,
             [GEN_AI_ATTRS.OPERATION]: "chat",
+            ...(sessionId !== undefined && { [GEN_AI_ATTRS.SESSION_ID]: sessionId }),
         });
+        let response;
+        try {
+            response = await original.call(thisArg, body, ...rest);
+        }
+        catch (err) {
+            const duration = performance.now() - start;
+            const message = err instanceof Error ? err.message : String(err);
+            // Release budget reservation — no cost was incurred
+            if (budget)
+                budget.releaseReservation(estimatedCost);
+            span.setAttributes({
+                [GEN_AI_ATTRS.STATUS]: "error",
+                [GEN_AI_ATTRS.ERROR]: message,
+            });
+            span.setStatus({ code: SpanStatusCode.ERROR, message });
+            span.end();
+            recordRequest(effectiveProvider, effectiveModel);
+            recordRequestDuration(duration, effectiveProvider, effectiveModel);
+            recordError(effectiveProvider, effectiveModel);
+            throw err;
+        }
         // Some SDKs (Gemini) return { stream: AsyncIterable } instead of a direct AsyncIterable
         const resp = response;
         const hasStreamProp = resp != null &&
@@ -87,13 +127,16 @@ original, patch, providerName) {
             ? resp["stream"]
             : response;
         const wrapped = context.bind(ctx, wrapAsyncIterable(streamIterable, (acc, chunk) => patch.accumulateChunk(acc, chunk), () => {
-            const ttft = performance.now() - start;
-            recordTimeToFirstToken(ttft, providerName, req.model);
+            recordTimeToFirstToken(performance.now() - start, effectiveProvider, effectiveModel);
         }, (acc) => {
             const duration = performance.now() - start;
-            const cost = calculateCost(req.model, acc.inputTokens, acc.outputTokens);
+            const cost = calculateCost(effectiveModel, acc.inputTokens, acc.outputTokens);
+            const processedCompletion = processContent(acc.completion);
             span.setAttributes({
-                [GEN_AI_ATTRS.RESPONSE_MODEL]: req.model,
+                ...(processedCompletion !== undefined && {
+                    [GEN_AI_ATTRS.COMPLETION]: processedCompletion,
+                }),
+                [GEN_AI_ATTRS.RESPONSE_MODEL]: effectiveModel,
                 [GEN_AI_ATTRS.INPUT_TOKENS]: acc.inputTokens,
                 [GEN_AI_ATTRS.OUTPUT_TOKENS]: acc.outputTokens,
                 [GEN_AI_ATTRS.COST]: cost,
@@ -102,22 +145,40 @@ original, patch, providerName) {
             });
             span.setStatus({ code: SpanStatusCode.OK });
             span.end();
-            recordRequest(providerName, req.model);
-            recordRequestDuration(duration, providerName, req.model);
-            recordRequestCost(cost, providerName, req.model);
-            recordTokens(acc.inputTokens + acc.outputTokens, providerName, req.model);
+            recordRequest(effectiveProvider, effectiveModel);
+            recordRequestDuration(duration, effectiveProvider, effectiveModel);
+            recordRequestCost(cost, effectiveProvider, effectiveModel);
+            recordTokens(acc.inputTokens + acc.outputTokens, effectiveProvider, effectiveModel);
+            // Quality metrics
+            if (acc.completion.trim() === "") {
+                recordResponseEmpty(effectiveProvider, effectiveModel);
+            }
+            if (acc.outputTokens > 0) {
+                recordResponseLatencyPerToken(duration / acc.outputTokens, effectiveProvider, effectiveModel);
+            }
+            // Budget recording — releases the reservation made in checkBefore
+            if (budget) {
+                const exceeded = budget.recordCost(cost, effectiveModel, userId, estimatedCost);
+                if (exceeded) {
+                    recordBudgetExceeded(exceeded.budget);
+                    console.warn(`toad-eye: ${exceeded.budget} budget exceeded — limit $${exceeded.limit}, current $${exceeded.current.toFixed(2)}`);
+                }
+            }
         }, (err) => {
             const duration = performance.now() - start;
             const message = err instanceof Error ? err.message : String(err);
+            // Release budget reservation on stream error
+            if (budget)
+                budget.releaseReservation(estimatedCost);
             span.setAttributes({
                 [GEN_AI_ATTRS.STATUS]: "error",
                 [GEN_AI_ATTRS.ERROR]: message,
             });
             span.setStatus({ code: SpanStatusCode.ERROR, message });
             span.end();
-            recordRequest(providerName, req.model);
-            recordRequestDuration(duration, providerName, req.model);
-            recordError(providerName, req.model);
+            recordRequest(effectiveProvider, effectiveModel);
+            recordRequestDuration(duration, effectiveProvider, effectiveModel);
+            recordError(effectiveProvider, effectiveModel);
         }));
         // Proxy preserves the original object shape (getters, private fields, methods)
         // while intercepting only the async iterator for instrumentation
@@ -144,7 +205,7 @@ original, patch, providerName) {
         if (streamHandler && patch.isStreaming?.(body)) {
             return streamHandler(this, body, rest);
         }
-        const req = patch.extractRequest(body);
+        const req = patch.extractRequest(body, this);
         return traceLLMCall({
             provider: providerName,
             model: req.model,

package/dist/instrumentations/gemini.d.ts

@@ -1 +1 @@
-export {};
+export declare const geminiInstrumentation: import("./types.js").Instrumentation;

package/dist/instrumentations/gemini.js

@@ -1,4 +1,3 @@
-import { register } from "./registry.js";
 import { createInstrumentation } from "./create.js";
 function extractPrompt(request) {
     if (typeof request === "string")
@@ -18,10 +17,10 @@ function extractPrompt(request) {
 const generateContent = {
     getPrototype: (sdk) => sdk?.GenerativeModel?.prototype,
     method: "generateContent",
-    extractRequest(body) {
+    extractRequest(body, thisArg) {
         return {
             prompt: extractPrompt(body),
-            model: "unknown",
+            model: thisArg?.model ?? "unknown",
         };
     },
     extractResponse: (response) => {
@@ -43,10 +42,10 @@ const generateContent = {
 const generateContentStream = {
     getPrototype: (sdk) => sdk?.GenerativeModel?.prototype,
     method: "generateContentStream",
-    extractRequest(body) {
+    extractRequest(body, thisArg) {
         return {
             prompt: extractPrompt(body),
-            model: "unknown",
+            model: thisArg?.model ?? "unknown",
         };
     },
     extractResponse: () => ({
@@ -74,8 +73,8 @@ const generateContentStream = {
         }
     },
 };
-register(createInstrumentation({
+export const geminiInstrumentation = createInstrumentation({
     name: "gemini",
     moduleName: "@google/generative-ai",
     patches: [generateContent, generateContentStream],
-}));
+});

package/dist/instrumentations/openai.d.ts

@@ -1 +1 @@
-export {};
+export declare const openaiInstrumentation: import("./types.js").Instrumentation;

package/dist/instrumentations/openai.js

@@ -1,4 +1,3 @@
-import { register } from "./registry.js";
 import { createInstrumentation } from "./create.js";
 function extractMessages(messages) {
     if (!Array.isArray(messages))
@@ -79,8 +78,8 @@ const embeddings = {
         };
     },
 };
-register(createInstrumentation({
+export const openaiInstrumentation = createInstrumentation({
     name: "openai",
     moduleName: "openai",
     patches: [chatCompletions, embeddings],
-}));
+});

package/dist/instrumentations/registry.d.ts

@@ -1,5 +1,7 @@
 import type { LLMProvider } from "../types/index.js";
 import type { Instrumentation } from "./types.js";
 export declare function register(inst: Instrumentation): void;
-export declare function enableAll(providers: readonly LLMProvider[]): void;
+/** Skip lazy registration (used in tests that register mocks manually). */
+export declare function skipAutoRegister(): void;
+export declare function enableAll(providers: readonly LLMProvider[]): Promise<void>;
 export declare function disableAll(): void;