toad-eye 2.0.0 → 2.1.1

package/dist/core/tracer.js

@@ -0,0 +1,93 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { initMetrics } from "./metrics.js";
+import { enableAll, disableAll } from "../instrumentations/registry.js";
+import { BudgetTracker } from "../budget/index.js";
+// Side-effect imports: register provider instrumentations
+import "../instrumentations/openai.js";
+import "../instrumentations/anthropic.js";
+import "../instrumentations/gemini.js";
+const DEFAULT_ENDPOINT = "http://localhost:4318";
+const DEFAULT_CLOUD_ENDPOINT = "https://cloud.toad-eye.dev";
+let sdk = null;
+let currentConfig = null;
+let budgetTracker = null;
+export function getConfig() {
+    return currentConfig;
+}
+export function getBudgetTracker() {
+    return budgetTracker;
+}
+function validateConfig(config) {
+    if (!config.serviceName?.trim()) {
+        throw new Error("toad-eye: serviceName is required and cannot be empty");
+    }
+    if (config.endpoint !== undefined && !config.endpoint.startsWith("http")) {
+        throw new Error(`toad-eye: endpoint must be a valid URL, got "${config.endpoint}"`);
+    }
+    if (config.apiKey !== undefined && !config.apiKey.startsWith("toad_")) {
+        throw new Error('toad-eye: apiKey must start with "toad_" prefix');
+    }
+}
+/** Resolve endpoint and auth headers based on config mode (self-hosted vs cloud). */
+function resolveTransport(config) {
+    const isCloudMode = config.apiKey !== undefined;
+    const endpoint = isCloudMode
+        ? (config.cloudEndpoint ?? DEFAULT_CLOUD_ENDPOINT)
+        : (config.endpoint ?? DEFAULT_ENDPOINT);
+    const headers = isCloudMode
+        ? { Authorization: `Bearer ${config.apiKey}` }
+        : {};
+    return { endpoint, headers, isCloudMode };
+}
+export function initObservability(config) {
+    if (sdk)
+        return;
+    validateConfig(config);
+    const { endpoint, headers, isCloudMode } = resolveTransport(config);
+    const resource = resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: config.serviceName,
+    });
+    const traceExporter = new OTLPTraceExporter({
+        url: `${endpoint}/v1/traces`,
+        headers,
+    });
+    const metricExporter = new OTLPMetricExporter({
+        url: `${endpoint}/v1/metrics`,
+        headers,
+    });
+    const metricReader = new PeriodicExportingMetricReader({
+        exporter: metricExporter,
+        exportIntervalMillis: isCloudMode ? 10_000 : 5_000,
+    });
+    sdk = new NodeSDK({
+        resource,
+        traceExporter,
+        metricReader,
+    });
+    sdk.start();
+    currentConfig = config;
+    initMetrics();
+    if (isCloudMode) {
+        console.log(`toad-eye: cloud mode enabled → sending telemetry to ${endpoint}`);
+    }
+    if (config.budgets) {
+        budgetTracker = new BudgetTracker(config.budgets, config.onBudgetExceeded ?? "warn", config.downgradeCallback);
+    }
+    if (config.instrument?.length) {
+        enableAll(config.instrument);
+    }
+}
+export async function shutdown() {
+    if (!sdk)
+        return;
+    disableAll();
+    await sdk.shutdown();
+    sdk = null;
+    currentConfig = null;
+    budgetTracker = null;
+}

package/dist/drift/baseline.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Baseline storage — save and load embedding baselines as JSON files.
+ *
+ * Baseline files contain embeddings of "known good" responses.
+ * Drift monitoring compares current responses against these baselines.
+ */
+export interface DriftBaseline {
+    readonly model: string;
+    readonly provider: string;
+    readonly embeddingModel: string;
+    readonly embeddings: readonly (readonly number[])[];
+    readonly sampleCount: number;
+    readonly createdAt: string;
+}
+export declare function saveBaseline(filepath: string, baseline: DriftBaseline): void;
+export declare function loadBaseline(filepath: string): DriftBaseline | undefined;

package/dist/drift/baseline.js

@@ -0,0 +1,16 @@
+/**
+ * Baseline storage — save and load embedding baselines as JSON files.
+ *
+ * Baseline files contain embeddings of "known good" responses.
+ * Drift monitoring compares current responses against these baselines.
+ */
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+export function saveBaseline(filepath, baseline) {
+    writeFileSync(filepath, JSON.stringify(baseline, null, 2), "utf-8");
+}
+export function loadBaseline(filepath) {
+    if (!existsSync(filepath))
+        return undefined;
+    const raw = readFileSync(filepath, "utf-8");
+    return JSON.parse(raw);
+}

package/dist/drift/cosine.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Cosine similarity and drift calculation for embedding vectors.
+ *
+ * drift = 1 - cosine_similarity(current, baseline)
+ * - drift = 0 → identical to baseline
+ * - drift > 0.3 → significant deviation (default alert threshold)
+ */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/**
+ * Calculate drift as distance from the nearest baseline embedding.
+ * Returns the minimum drift value (closest match).
+ */
+export declare function cosineDrift(current: readonly number[], baselines: readonly (readonly number[])[]): number;

package/dist/drift/cosine.js

@@ -0,0 +1,39 @@
+/**
+ * Cosine similarity and drift calculation for embedding vectors.
+ *
+ * drift = 1 - cosine_similarity(current, baseline)
+ * - drift = 0 → identical to baseline
+ * - drift > 0.3 → significant deviation (default alert threshold)
+ */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const ai = a[i];
+        const bi = b[i];
+        dot += ai * bi;
+        normA += ai * ai;
+        normB += bi * bi;
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    if (denominator === 0)
+        return 0;
+    return dot / denominator;
+}
+/**
+ * Calculate drift as distance from the nearest baseline embedding.
+ * Returns the minimum drift value (closest match).
+ */
+export function cosineDrift(current, baselines) {
+    if (baselines.length === 0)
+        return 1;
+    let maxSimilarity = -1;
+    for (const baseline of baselines) {
+        const similarity = cosineSimilarity(current, baseline);
+        if (similarity > maxSimilarity) {
+            maxSimilarity = similarity;
+        }
+    }
+    return 1 - maxSimilarity;
+}

package/dist/drift/embedding-provider.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Embedding provider abstraction for semantic drift monitoring.
+ *
+ * Implementations convert text into float vectors for cosine similarity comparison.
+ * Default: OpenAI text-embedding-3-small (cheap, fast).
+ * Users can provide a custom implementation via the EmbeddingProvider interface.
+ */
+export interface EmbeddingProvider {
+    readonly name: string;
+    embed(text: string): Promise<readonly number[]>;
+}
+export interface OpenAIEmbeddingConfig {
+    readonly provider: "openai";
+    readonly apiKey: string;
+    readonly model?: string | undefined;
+}
+export interface CustomEmbeddingConfig {
+    readonly provider: "custom";
+    readonly embed: (text: string) => Promise<readonly number[]>;
+}
+export type EmbeddingConfig = OpenAIEmbeddingConfig | CustomEmbeddingConfig;

package/dist/drift/embedding-provider.js

@@ -0,0 +1,8 @@
+/**
+ * Embedding provider abstraction for semantic drift monitoring.
+ *
+ * Implementations convert text into float vectors for cosine similarity comparison.
+ * Default: OpenAI text-embedding-3-small (cheap, fast).
+ * Users can provide a custom implementation via the EmbeddingProvider interface.
+ */
+export {};

package/dist/drift/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Semantic Drift Monitoring — detect silent LLM quality degradation
+ * by comparing current responses to a saved baseline via embeddings.
+ */
+export type { EmbeddingProvider, EmbeddingConfig, OpenAIEmbeddingConfig, CustomEmbeddingConfig, } from "./embedding-provider.js";
+export { createOpenAIEmbeddingProvider } from "./openai-embeddings.js";
+export { cosineSimilarity, cosineDrift } from "./cosine.js";
+export { saveBaseline, loadBaseline, type DriftBaseline } from "./baseline.js";
+export { createDriftMonitor, type DriftMonitor, type DriftMonitorConfig, } from "./monitor.js";

package/dist/drift/index.js

@@ -0,0 +1,8 @@
+/**
+ * Semantic Drift Monitoring — detect silent LLM quality degradation
+ * by comparing current responses to a saved baseline via embeddings.
+ */
+export { createOpenAIEmbeddingProvider } from "./openai-embeddings.js";
+export { cosineSimilarity, cosineDrift } from "./cosine.js";
+export { saveBaseline, loadBaseline } from "./baseline.js";
+export { createDriftMonitor, } from "./monitor.js";

package/dist/drift/monitor.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Drift monitor — runtime semantic drift detection with sampling.
+ *
+ * Not every response is checked (embedding API calls cost money).
+ * By default, every 10th response is sampled. The drift value is
+ * recorded as a Prometheus histogram metric with provider/model labels.
+ */
+import type { EmbeddingConfig } from "./embedding-provider.js";
+export interface DriftMonitorConfig {
+    readonly embedding: EmbeddingConfig;
+    readonly baselinePath: string;
+    readonly sampleRate?: number | undefined;
+}
+export interface DriftMonitor {
+    /**
+     * Check a response for semantic drift. Respects sampling rate —
+     * only every Nth call actually computes embeddings.
+     * Returns the drift value if checked, undefined if skipped.
+     */
+    check(response: string, provider: string, model: string): Promise<number | undefined>;
+    /**
+     * Fire-and-forget drift check — does NOT block the caller.
+     * Errors are logged to console.warn, never thrown.
+     * Use this in hot paths (e.g. inside traceLLMCall) to avoid
+     * blocking the LLM response delivery.
+     */
+    checkInBackground(response: string, provider: string, model: string): void;
+}
+export declare function createDriftMonitor(config: DriftMonitorConfig): DriftMonitor;

package/dist/drift/monitor.js

@@ -0,0 +1,47 @@
+/**
+ * Drift monitor — runtime semantic drift detection with sampling.
+ *
+ * Not every response is checked (embedding API calls cost money).
+ * By default, every 10th response is sampled. The drift value is
+ * recorded as a Prometheus histogram metric with provider/model labels.
+ */
+import { createOpenAIEmbeddingProvider } from "./openai-embeddings.js";
+import { cosineDrift } from "./cosine.js";
+import { loadBaseline } from "./baseline.js";
+import { recordSemanticDrift } from "../core/metrics.js";
+function createProvider(config) {
+    switch (config.provider) {
+        case "openai":
+            return createOpenAIEmbeddingProvider(config.apiKey, config.model);
+        case "custom":
+            return { name: "custom", embed: config.embed };
+    }
+}
+export function createDriftMonitor(config) {
+    const provider = createProvider(config.embedding);
+    const sampleRate = config.sampleRate ?? 10;
+    const baseline = loadBaseline(config.baselinePath);
+    let counter = 0;
+    if (baseline === undefined) {
+        console.warn(`[toad-eye] No baseline found at ${config.baselinePath}. Drift monitoring disabled until baseline is saved.`);
+    }
+    async function performCheck(response, providerName, model) {
+        if (baseline === undefined)
+            return undefined;
+        counter++;
+        if (counter % sampleRate !== 0)
+            return undefined;
+        const embedding = await provider.embed(response);
+        const drift = cosineDrift(embedding, baseline.embeddings);
+        recordSemanticDrift(drift, providerName, model);
+        return drift;
+    }
+    return {
+        check: performCheck,
+        checkInBackground(response, providerName, model) {
+            void performCheck(response, providerName, model).catch((err) => {
+                console.warn(`[toad-eye] Drift check failed (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
+            });
+        },
+    };
+}

package/dist/drift/openai-embeddings.d.ts

@@ -0,0 +1,6 @@
+/**
+ * OpenAI embedding provider using text-embedding-3-small.
+ * Calls the OpenAI API directly via fetch — no SDK dependency.
+ */
+import type { EmbeddingProvider } from "./embedding-provider.js";
+export declare function createOpenAIEmbeddingProvider(apiKey: string, model?: string): EmbeddingProvider;

package/dist/drift/openai-embeddings.js

@@ -0,0 +1,26 @@
+/**
+ * OpenAI embedding provider using text-embedding-3-small.
+ * Calls the OpenAI API directly via fetch — no SDK dependency.
+ */
+const DEFAULT_MODEL = "text-embedding-3-small";
+const OPENAI_API_URL = "https://api.openai.com/v1/embeddings";
+export function createOpenAIEmbeddingProvider(apiKey, model = DEFAULT_MODEL) {
+    return {
+        name: `openai/${model}`,
+        async embed(text) {
+            const response = await fetch(OPENAI_API_URL, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify({ input: text, model }),
+            });
+            if (!response.ok) {
+                throw new Error(`OpenAI Embeddings API error: ${response.status} ${response.statusText}`);
+            }
+            const body = (await response.json());
+            return body.data[0].embedding;
+        },
+    };
+}

package/dist/export.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Trace-to-Dataset Export — converts production Jaeger traces into toad-eval test cases.
+ *
+ * Fetches a trace by ID from the Jaeger Query API, extracts LLM call data
+ * (prompt, model, provider, response), and generates a YAML file in toad-eval format
+ * with auto-generated assertions (max_length, not_contains, is_json).
+ */
+interface JaegerTag {
+    readonly key: string;
+    readonly type: string;
+    readonly value: string | number | boolean;
+}
+interface JaegerSpan {
+    readonly traceID: string;
+    readonly spanID: string;
+    readonly operationName: string;
+    readonly tags: readonly JaegerTag[];
+    readonly startTime: number;
+    readonly duration: number;
+}
+interface JaegerTrace {
+    readonly traceID: string;
+    readonly spans: readonly JaegerSpan[];
+}
+export interface ExportTraceOptions {
+    readonly jaegerUrl?: string | undefined;
+    readonly output?: string | undefined;
+}
+/**
+ * Fetch a trace from Jaeger by trace ID.
+ * Requires Jaeger to be running with the Query API available.
+ */
+export declare function fetchTrace(traceId: string, jaegerUrl?: string): Promise<JaegerTrace>;
+/**
+ * Convert a Jaeger trace to a toad-eval YAML string.
+ * Extracts LLM spans with prompt data, generates assertions from completions.
+ *
+ * Returns undefined if no exportable spans found (e.g. recordContent was false).
+ */
+export declare function traceToEvalYaml(trace: JaegerTrace): string | undefined;
+/**
+ * Full export pipeline: fetch trace from Jaeger → convert to toad-eval YAML.
+ * Returns the YAML string. Caller is responsible for writing to file.
+ */
+export declare function exportTrace(traceId: string, options?: ExportTraceOptions): Promise<string>;
+export {};

package/dist/export.js

@@ -0,0 +1,108 @@
+/**
+ * Trace-to-Dataset Export — converts production Jaeger traces into toad-eval test cases.
+ *
+ * Fetches a trace by ID from the Jaeger Query API, extracts LLM call data
+ * (prompt, model, provider, response), and generates a YAML file in toad-eval format
+ * with auto-generated assertions (max_length, not_contains, is_json).
+ */
+import { stringify } from "yaml";
+import { GEN_AI_ATTRS } from "./types/index.js";
+const DEFAULT_JAEGER_URL = "http://localhost:16686";
+const LENGTH_BUFFER_MULTIPLIER = 1.5;
+const REFUSAL_MARKER = "i cannot";
+function getTagValue(tags, key) {
+    const tag = tags.find((t) => t.key === key);
+    return tag !== undefined ? String(tag.value) : undefined;
+}
+function isJson(text) {
+    try {
+        JSON.parse(text);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function buildAssertions(completion) {
+    const assertions = [];
+    if (completion !== undefined) {
+        assertions.push({
+            type: "max_length",
+            value: Math.ceil(completion.length * LENGTH_BUFFER_MULTIPLIER),
+        });
+        if (!completion.toLowerCase().includes(REFUSAL_MARKER)) {
+            assertions.push({ type: "not_contains", value: "I cannot" });
+        }
+        if (isJson(completion)) {
+            assertions.push({ type: "is_json", value: true });
+        }
+    }
+    return assertions;
+}
+function spanToEvalCase(span, index) {
+    const prompt = getTagValue(span.tags, GEN_AI_ATTRS.PROMPT);
+    if (prompt === undefined)
+        return undefined;
+    const completion = getTagValue(span.tags, GEN_AI_ATTRS.COMPLETION);
+    return {
+        id: `production-case-${index + 1}`,
+        variables: { input: prompt },
+        assertions: buildAssertions(completion),
+    };
+}
+/**
+ * Fetch a trace from Jaeger by trace ID.
+ * Requires Jaeger to be running with the Query API available.
+ */
+export async function fetchTrace(traceId, jaegerUrl = DEFAULT_JAEGER_URL) {
+    const url = `${jaegerUrl}/api/traces/${traceId}`;
+    const response = await fetch(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch trace ${traceId}: ${response.status} ${response.statusText}`);
+    }
+    const body = (await response.json());
+    if (body.data.length === 0) {
+        throw new Error(`Trace ${traceId} not found`);
+    }
+    return body.data[0];
+}
+/**
+ * Convert a Jaeger trace to a toad-eval YAML string.
+ * Extracts LLM spans with prompt data, generates assertions from completions.
+ *
+ * Returns undefined if no exportable spans found (e.g. recordContent was false).
+ */
+export function traceToEvalYaml(trace) {
+    const cases = trace.spans
+        .map((span, i) => spanToEvalCase(span, i))
+        .filter((c) => c !== undefined);
+    if (cases.length === 0)
+        return undefined;
+    // Pick model/provider from first span that has them
+    const firstSpanWithMeta = trace.spans.find((s) => getTagValue(s.tags, GEN_AI_ATTRS.REQUEST_MODEL) !== undefined);
+    const dataset = {
+        name: `exported-trace-${trace.traceID.slice(0, 8)}`,
+        source: "toad-eye-export",
+        metadata: {
+            trace_id: trace.traceID,
+            exported_at: new Date().toISOString(),
+            model: getTagValue(firstSpanWithMeta?.tags ?? [], GEN_AI_ATTRS.REQUEST_MODEL) ?? "unknown",
+            provider: getTagValue(firstSpanWithMeta?.tags ?? [], GEN_AI_ATTRS.PROVIDER) ??
+                "unknown",
+        },
+        cases,
+    };
+    return stringify(dataset);
+}
+/**
+ * Full export pipeline: fetch trace from Jaeger → convert to toad-eval YAML.
+ * Returns the YAML string. Caller is responsible for writing to file.
+ */
+export async function exportTrace(traceId, options = {}) {
+    const trace = await fetchTrace(traceId, options.jaegerUrl);
+    const yaml = traceToEvalYaml(trace);
+    if (yaml === undefined) {
+        throw new Error(`No exportable spans in trace ${traceId}. Was recordContent enabled?`);
+    }
+    return yaml;
+}

package/dist/guard.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Shadow Guardrails integration — records toad-guard validation results
+ * as span attributes and metrics without blocking LLM responses.
+ *
+ * toad-guard validates each LLM response and passes the result here.
+ * In shadow mode, validation runs but never throws — results are recorded
+ * for observability only, letting you tune guardrail thresholds on
+ * production traffic before switching to enforce mode.
+ *
+ * Usage from toad-guard:
+ * ```ts
+ * import { recordGuardResult } from "toad-eye";
+ *
+ * const result = guard.validate(response);
+ * recordGuardResult(result); // records as span attrs + metrics
+ * ```
+ */
+import type { GuardResult } from "./types/index.js";
+/**
+ * Record a guard validation result as span attributes and metrics.
+ *
+ * Attaches guard attributes to a child span and increments counters.
+ * If the guard failed, also increments the `would_block` counter —
+ * useful for measuring how often shadow guardrails would have blocked.
+ */
+export declare function recordGuardResult(result: GuardResult): void;

package/dist/guard.js

@@ -0,0 +1,45 @@
+/**
+ * Shadow Guardrails integration — records toad-guard validation results
+ * as span attributes and metrics without blocking LLM responses.
+ *
+ * toad-guard validates each LLM response and passes the result here.
+ * In shadow mode, validation runs but never throws — results are recorded
+ * for observability only, letting you tune guardrail thresholds on
+ * production traffic before switching to enforce mode.
+ *
+ * Usage from toad-guard:
+ * ```ts
+ * import { recordGuardResult } from "toad-eye";
+ *
+ * const result = guard.validate(response);
+ * recordGuardResult(result); // records as span attrs + metrics
+ * ```
+ */
+import { trace } from "@opentelemetry/api";
+import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "./types/index.js";
+import { recordGuardEvaluation, recordGuardWouldBlock, } from "./core/metrics.js";
+const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+/**
+ * Record a guard validation result as span attributes and metrics.
+ *
+ * Attaches guard attributes to a child span and increments counters.
+ * If the guard failed, also increments the `would_block` counter —
+ * useful for measuring how often shadow guardrails would have blocked.
+ */
+export function recordGuardResult(result) {
+    const span = tracer.startSpan(`guard.evaluate.${result.ruleName}`);
+    span.setAttributes({
+        [GEN_AI_ATTRS.GUARD_MODE]: result.mode,
+        [GEN_AI_ATTRS.GUARD_PASSED]: result.passed,
+        [GEN_AI_ATTRS.GUARD_RULE_NAME]: result.ruleName,
+        ...(!result.passed &&
+            result.failureReason !== undefined && {
+            [GEN_AI_ATTRS.GUARD_FAILURE_REASON]: result.failureReason,
+        }),
+    });
+    span.end();
+    recordGuardEvaluation(result.ruleName);
+    if (!result.passed) {
+        recordGuardWouldBlock(result.ruleName);
+    }
+}

package/dist/index.d.ts

@@ -1,9 +1,17 @@
-export { initObservability, shutdown, getConfig } from "./tracer.js";
+export { initObservability, shutdown, getConfig, getBudgetTracker, } from "./core/tracer.js";
+export { BudgetTracker, ToadBudgetExceededError } from "./budget/index.js";
+export type { BudgetConfig, BudgetExceededMode, BudgetExceededInfo, DowngradeCallback, } from "./budget/index.js";
 export { AlertManager, startAlertsFromFile, parseCondition, } from "./alerts/index.js";
 export type { AlertsConfig, AlertRule, AlertChannelConfig, FiredAlert, } from "./alerts/index.js";
-export { traceLLMCall } from "./spans.js";
-export { calculateCost, setCustomPricing, getModelPricing } from "./pricing.js";
-export type { ModelPricing } from "./pricing.js";
-export type { LLMCallInput, LLMCallOutput } from "./spans.js";
-export type { ToadEyeConfig, LLMSpanAttributes, LLMProvider, SpanStatus, MetricName, } from "./types/index.js";
+export { traceLLMCall } from "./core/spans.js";
+export { traceAgentStep, traceAgentQuery } from "./agent.js";
+export { recordGuardResult } from "./guard.js";
+export { createDriftMonitor, createOpenAIEmbeddingProvider, cosineSimilarity, cosineDrift, saveBaseline, loadBaseline, } from "./drift/index.js";
+export type { DriftMonitor, DriftMonitorConfig, EmbeddingProvider, EmbeddingConfig, DriftBaseline, } from "./drift/index.js";
+export { exportTrace, fetchTrace, traceToEvalYaml } from "./export.js";
+export type { ExportTraceOptions } from "./export.js";
+export { calculateCost, setCustomPricing, getModelPricing, } from "./core/pricing.js";
+export type { ModelPricing } from "./core/pricing.js";
+export type { LLMCallInput, LLMCallOutput } from "./core/spans.js";
+export type { ToadEyeConfig, LLMSpanAttributes, LLMProvider, SpanStatus, MetricName, AgentStepType, AgentStepInput, GuardMode, GuardResult, } from "./types/index.js";
 export { GEN_AI_METRICS, GEN_AI_ATTRS, LLM_METRICS, LLM_ATTRS, } from "./types/index.js";

package/dist/index.js

@@ -1,5 +1,10 @@
-export { initObservability, shutdown, getConfig } from "./tracer.js";
+export { initObservability, shutdown, getConfig, getBudgetTracker, } from "./core/tracer.js";
+export { BudgetTracker, ToadBudgetExceededError } from "./budget/index.js";
 export { AlertManager, startAlertsFromFile, parseCondition, } from "./alerts/index.js";
-export { traceLLMCall } from "./spans.js";
-export { calculateCost, setCustomPricing, getModelPricing } from "./pricing.js";
+export { traceLLMCall } from "./core/spans.js";
+export { traceAgentStep, traceAgentQuery } from "./agent.js";
+export { recordGuardResult } from "./guard.js";
+export { createDriftMonitor, createOpenAIEmbeddingProvider, cosineSimilarity, cosineDrift, saveBaseline, loadBaseline, } from "./drift/index.js";
+export { exportTrace, fetchTrace, traceToEvalYaml } from "./export.js";
+export { calculateCost, setCustomPricing, getModelPricing, } from "./core/pricing.js";
 export { GEN_AI_METRICS, GEN_AI_ATTRS, LLM_METRICS, LLM_ATTRS, } from "./types/index.js";

package/dist/instrumentations/anthropic.js

@@ -29,7 +29,6 @@ function extractCompletion(content) {
 const messagesCreate = {
     getPrototype: (sdk) => sdk?.Messages?.prototype,
     method: "create",
-    shouldSkip: (body) => !!body?.stream,
     extractRequest: (body) => {
         const b = body;
         return {
@@ -46,6 +45,26 @@ const messagesCreate = {
             outputTokens: r?.usage?.output_tokens ?? 0,
         };
     },
+    isStreaming: (body) => !!body?.stream,
+    extractStreamResponse: (chunks) => {
+        // Anthropic stream events: message_start, content_block_delta, message_delta, message_stop
+        let completion = "";
+        let inputTokens = 0;
+        let outputTokens = 0;
+        for (const chunk of chunks) {
+            const event = chunk;
+            if (event.type === "content_block_delta" && event.delta?.text) {
+                completion += event.delta.text;
+            }
+            if (event.type === "message_start" && event.message?.usage) {
+                inputTokens = event.message.usage.input_tokens ?? 0;
+            }
+            if (event.type === "message_delta" && event.usage) {
+                outputTokens = event.usage.output_tokens ?? 0;
+            }
+        }
+        return { completion, inputTokens, outputTokens };
+    },
 };
 register(createInstrumentation({
     name: "anthropic",