toad-eye 1.1.0 → 2.1.0

package/dist/core/metrics.js

@@ -0,0 +1,119 @@
+import { metrics } from "@opentelemetry/api";
+import { GEN_AI_ATTRS, GEN_AI_METRICS, INSTRUMENTATION_NAME, } from "../types/index.js";
+let requestDuration;
+let requestCost;
+let tokenUsage;
+let requestsTotal;
+let errorsTotal;
+let agentStepsPerQuery;
+let agentToolUsage;
+let guardEvaluations;
+let guardWouldBlock;
+let semanticDrift;
+let budgetExceeded;
+let budgetBlocked;
+let budgetDowngraded;
+let initialized = false;
+export function initMetrics() {
+    if (initialized)
+        return;
+    const meter = metrics.getMeter(INSTRUMENTATION_NAME);
+    requestDuration = meter.createHistogram(GEN_AI_METRICS.REQUEST_DURATION, {
+        description: "GenAI operation duration in milliseconds",
+        unit: "ms",
+    });
+    requestCost = meter.createHistogram(GEN_AI_METRICS.REQUEST_COST, {
+        description: "GenAI request cost in USD",
+        unit: "USD",
+    });
+    tokenUsage = meter.createCounter(GEN_AI_METRICS.TOKEN_USAGE, {
+        description: "Total tokens consumed",
+    });
+    requestsTotal = meter.createCounter(GEN_AI_METRICS.REQUESTS, {
+        description: "Total GenAI requests",
+    });
+    errorsTotal = meter.createCounter(GEN_AI_METRICS.ERRORS, {
+        description: "Total failed GenAI requests",
+    });
+    agentStepsPerQuery = meter.createHistogram(GEN_AI_METRICS.AGENT_STEPS_PER_QUERY, {
+        description: "Number of agent steps per query",
+    });
+    agentToolUsage = meter.createCounter(GEN_AI_METRICS.AGENT_TOOL_USAGE, {
+        description: "Agent tool invocations by tool name",
+    });
+    guardEvaluations = meter.createCounter(GEN_AI_METRICS.GUARD_EVALUATIONS, {
+        description: "Total guard evaluations (shadow + enforce)",
+    });
+    guardWouldBlock = meter.createCounter(GEN_AI_METRICS.GUARD_WOULD_BLOCK, {
+        description: "Guard evaluations that would have blocked the response",
+    });
+    semanticDrift = meter.createHistogram(GEN_AI_METRICS.SEMANTIC_DRIFT, {
+        description: "Semantic drift from baseline (0 = identical, 1 = completely different)",
+    });
+    budgetExceeded = meter.createCounter(GEN_AI_METRICS.BUDGET_EXCEEDED, {
+        description: "Number of times a budget limit was exceeded",
+    });
+    budgetBlocked = meter.createCounter(GEN_AI_METRICS.BUDGET_BLOCKED, {
+        description: "Number of LLM calls blocked due to budget limits",
+    });
+    budgetDowngraded = meter.createCounter(GEN_AI_METRICS.BUDGET_DOWNGRADED, {
+        description: "Number of LLM calls downgraded due to budget limits",
+    });
+    initialized = true;
+}
+/** Build metric labels: provider + model + optional FinOps attributes. */
+function baseLabels(provider, model, attrs) {
+    return {
+        [GEN_AI_ATTRS.PROVIDER]: provider,
+        [GEN_AI_ATTRS.REQUEST_MODEL]: model,
+        ...attrs,
+    };
+}
+export function recordRequestDuration(ms, provider, model, attrs) {
+    requestDuration.record(ms, baseLabels(provider, model, attrs));
+}
+export function recordRequestCost(usd, provider, model, attrs) {
+    requestCost.record(usd, baseLabels(provider, model, attrs));
+}
+export function recordTokens(count, provider, model, attrs) {
+    tokenUsage.add(count, baseLabels(provider, model, attrs));
+}
+export function recordRequest(provider, model, attrs) {
+    requestsTotal.add(1, baseLabels(provider, model, attrs));
+}
+export function recordError(provider, model, attrs) {
+    errorsTotal.add(1, baseLabels(provider, model, attrs));
+}
+export function recordAgentSteps(stepCount) {
+    agentStepsPerQuery.record(stepCount);
+}
+export function recordAgentToolUsage(toolName) {
+    agentToolUsage.add(1, {
+        [GEN_AI_ATTRS.AGENT_TOOL_NAME]: toolName,
+    });
+}
+export function recordGuardEvaluation(ruleName) {
+    guardEvaluations.add(1, {
+        [GEN_AI_ATTRS.GUARD_RULE_NAME]: ruleName,
+    });
+}
+export function recordGuardWouldBlock(ruleName) {
+    guardWouldBlock.add(1, {
+        [GEN_AI_ATTRS.GUARD_RULE_NAME]: ruleName,
+    });
+}
+export function recordSemanticDrift(drift, provider, model) {
+    semanticDrift.record(drift, {
+        [GEN_AI_ATTRS.PROVIDER]: provider,
+        [GEN_AI_ATTRS.REQUEST_MODEL]: model,
+    });
+}
+export function recordBudgetExceeded(budgetType) {
+    budgetExceeded.add(1, { budget_type: budgetType });
+}
+export function recordBudgetBlocked(budgetType) {
+    budgetBlocked.add(1, { budget_type: budgetType });
+}
+export function recordBudgetDowngraded(budgetType) {
+    budgetDowngraded.add(1, { budget_type: budgetType });
+}

package/dist/core/pricing.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Built-in pricing table for major LLM models.
+ * Prices in USD per 1M tokens (input / output).
+ * Updated: March 2026.
+ */
+export interface ModelPricing {
+    readonly inputPer1M: number;
+    readonly outputPer1M: number;
+}
+/**
+ * Set custom pricing for models (overrides built-in prices).
+ * Useful for enterprise contracts or custom/fine-tuned models.
+ */
+export declare function setCustomPricing(pricing: Record<string, ModelPricing>): void;
+/**
+ * Get pricing for a model. Custom pricing takes precedence.
+ * Returns undefined if model is not in any pricing table.
+ */
+export declare function getModelPricing(model: string): ModelPricing | undefined;
+/**
+ * Calculate cost for a given model and token counts.
+ * Returns 0 if model pricing is not found.
+ */
+export declare function calculateCost(model: string, inputTokens: number, outputTokens: number): number;

package/dist/core/pricing.js

@@ -0,0 +1,52 @@
+/**
+ * Built-in pricing table for major LLM models.
+ * Prices in USD per 1M tokens (input / output).
+ * Updated: March 2026.
+ */
+const BUILT_IN_PRICING = {
+    // OpenAI
+    "gpt-4o": { inputPer1M: 2.5, outputPer1M: 10 },
+    "gpt-4o-mini": { inputPer1M: 0.15, outputPer1M: 0.6 },
+    "gpt-4-turbo": { inputPer1M: 10, outputPer1M: 30 },
+    "gpt-4.1": { inputPer1M: 2, outputPer1M: 8 },
+    "gpt-4.1-mini": { inputPer1M: 0.4, outputPer1M: 1.6 },
+    "gpt-4.1-nano": { inputPer1M: 0.1, outputPer1M: 0.4 },
+    o3: { inputPer1M: 10, outputPer1M: 40 },
+    "o3-mini": { inputPer1M: 1.1, outputPer1M: 4.4 },
+    "o4-mini": { inputPer1M: 1.1, outputPer1M: 4.4 },
+    // Anthropic
+    "claude-opus-4-20250514": { inputPer1M: 15, outputPer1M: 75 },
+    "claude-sonnet-4-20250514": { inputPer1M: 3, outputPer1M: 15 },
+    "claude-haiku-3-5-20241022": { inputPer1M: 0.8, outputPer1M: 4 },
+    // Google Gemini
+    "gemini-2.5-pro": { inputPer1M: 1.25, outputPer1M: 10 },
+    "gemini-2.5-flash": { inputPer1M: 0.15, outputPer1M: 0.6 },
+    "gemini-2.0-flash": { inputPer1M: 0.1, outputPer1M: 0.4 },
+};
+let customPricing = {};
+/**
+ * Set custom pricing for models (overrides built-in prices).
+ * Useful for enterprise contracts or custom/fine-tuned models.
+ */
+export function setCustomPricing(pricing) {
+    customPricing = { ...pricing };
+}
+/**
+ * Get pricing for a model. Custom pricing takes precedence.
+ * Returns undefined if model is not in any pricing table.
+ */
+export function getModelPricing(model) {
+    return customPricing[model] ?? BUILT_IN_PRICING[model];
+}
+/**
+ * Calculate cost for a given model and token counts.
+ * Returns 0 if model pricing is not found.
+ */
+export function calculateCost(model, inputTokens, outputTokens) {
+    const pricing = getModelPricing(model);
+    if (!pricing)
+        return 0;
+    const cost = (inputTokens / 1_000_000) * pricing.inputPer1M +
+        (outputTokens / 1_000_000) * pricing.outputPer1M;
+    return Math.round(cost * 1_000_000) / 1_000_000;
+}

package/dist/{spans.d.ts → core/spans.d.ts}

@@ -1,10 +1,12 @@
-import type { LLMSpanAttributes } from "./types.js";
+import type { LLMSpanAttributes } from "../types/index.js";
 /** Input for traceLLMCall — what the user knows before calling the LLM */
 export interface LLMCallInput {
     readonly provider: LLMSpanAttributes["provider"];
     readonly model: string;
     readonly prompt: string;
     readonly temperature?: number | undefined;
+    /** Per-request FinOps attributes (team, userId, feature, etc). Merged with global attributes. */
+    readonly attributes?: Readonly<Record<string, string>> | undefined;
 }
 /** Output from the LLM call — what comes back */
 export interface LLMCallOutput {

package/dist/core/spans.js

@@ -0,0 +1,143 @@
+import { createHash } from "node:crypto";
+import { trace, SpanStatusCode } from "@opentelemetry/api";
+import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "../types/index.js";
+import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordBudgetExceeded, recordBudgetBlocked, recordBudgetDowngraded, } from "./metrics.js";
+import { getConfig, getBudgetTracker } from "./tracer.js";
+import { GEN_AI_ATTRS as ATTRS } from "../types/index.js";
+const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+function sha256(text) {
+    const salt = getConfig()?.salt ?? "";
+    return createHash("sha256")
+        .update(salt + text)
+        .digest("hex");
+}
+function processContent(text) {
+    const config = getConfig();
+    if (config?.recordContent === false)
+        return undefined;
+    let processed = text;
+    if (config?.redactPatterns?.length) {
+        for (const pattern of config.redactPatterns) {
+            processed = processed.replace(pattern, "[REDACTED]");
+        }
+    }
+    if (config?.hashContent) {
+        return `sha256:${sha256(processed)}`;
+    }
+    return processed;
+}
+function resolveSessionId() {
+    const config = getConfig();
+    return config?.sessionExtractor?.() ?? config?.sessionId;
+}
+/** Merge global config attributes with per-request attributes (per-request wins). */
+function resolveAttributes(input) {
+    const global = getConfig()?.attributes;
+    const local = input.attributes;
+    if (!global && !local)
+        return undefined;
+    return { ...global, ...local };
+}
+function setBaseAttributes(span, input) {
+    const prompt = processContent(input.prompt);
+    const sessionId = resolveSessionId();
+    const attrs = resolveAttributes(input);
+    span.setAttributes({
+        [GEN_AI_ATTRS.PROVIDER]: input.provider,
+        [GEN_AI_ATTRS.REQUEST_MODEL]: input.model,
+        [GEN_AI_ATTRS.TEMPERATURE]: input.temperature ?? 1.0,
+        [GEN_AI_ATTRS.OPERATION]: "chat",
+        ...(prompt !== undefined && { [GEN_AI_ATTRS.PROMPT]: prompt }),
+        ...(sessionId !== undefined && { [GEN_AI_ATTRS.SESSION_ID]: sessionId }),
+        ...attrs,
+    });
+}
+function recordBaseMetrics(duration, provider, model, attrs) {
+    recordRequest(provider, model, attrs);
+    recordRequestDuration(duration, provider, model, attrs);
+}
+function setSuccessAttributes(span, input, output) {
+    const completion = processContent(output.completion);
+    span.setAttributes({
+        ...(completion !== undefined && { [GEN_AI_ATTRS.COMPLETION]: completion }),
+        [GEN_AI_ATTRS.RESPONSE_MODEL]: input.model,
+        [GEN_AI_ATTRS.INPUT_TOKENS]: output.inputTokens,
+        [GEN_AI_ATTRS.OUTPUT_TOKENS]: output.outputTokens,
+        [GEN_AI_ATTRS.COST]: output.cost,
+        [GEN_AI_ATTRS.STATUS]: "success",
+        [GEN_AI_ATTRS.FINISH_REASONS]: ["stop"],
+    });
+    span.setStatus({ code: SpanStatusCode.OK });
+}
+function setErrorAttributes(span, message) {
+    const sanitized = processContent(message);
+    span.setAttributes({
+        [GEN_AI_ATTRS.INPUT_TOKENS]: 0,
+        [GEN_AI_ATTRS.OUTPUT_TOKENS]: 0,
+        [GEN_AI_ATTRS.COST]: 0,
+        [GEN_AI_ATTRS.STATUS]: "error",
+        ...(sanitized !== undefined && { [GEN_AI_ATTRS.ERROR]: sanitized }),
+    });
+    span.setStatus({
+        code: SpanStatusCode.ERROR,
+        ...(sanitized !== undefined && { message: sanitized }),
+    });
+}
+export async function traceLLMCall(input, fn) {
+    const budget = getBudgetTracker();
+    let effectiveInput = input;
+    // Budget check BEFORE the LLM call
+    if (budget) {
+        const userId = input.attributes?.[ATTRS.USER_ID];
+        const override = budget.checkBefore(input.provider, input.model, userId);
+        if (override) {
+            // Downgrade mode — use modified provider/model
+            effectiveInput = {
+                ...input,
+                provider: override.provider,
+                model: override.model,
+            };
+            recordBudgetDowngraded("daily");
+        }
+    }
+    return tracer.startActiveSpan(`gen_ai.${effectiveInput.provider}.${effectiveInput.model}`, async (span) => {
+        const start = performance.now();
+        setBaseAttributes(span, effectiveInput);
+        try {
+            const output = await fn();
+            const duration = performance.now() - start;
+            const attrs = resolveAttributes(effectiveInput);
+            setSuccessAttributes(span, effectiveInput, output);
+            recordBaseMetrics(duration, effectiveInput.provider, effectiveInput.model, attrs);
+            recordRequestCost(output.cost, effectiveInput.provider, effectiveInput.model, attrs);
+            recordTokens(output.inputTokens + output.outputTokens, effectiveInput.provider, effectiveInput.model, attrs);
+            // Budget recording AFTER the call
+            if (budget) {
+                const userId = effectiveInput.attributes?.[ATTRS.USER_ID];
+                const exceeded = budget.recordCost(output.cost, effectiveInput.model, userId);
+                if (exceeded) {
+                    recordBudgetExceeded(exceeded.budget);
+                    console.warn(`toad-eye: ${exceeded.budget} budget exceeded — limit $${exceeded.limit}, current $${exceeded.current.toFixed(2)}`);
+                }
+            }
+            return output;
+        }
+        catch (error) {
+            const duration = performance.now() - start;
+            const message = error instanceof Error ? error.message : String(error);
+            const attrs = resolveAttributes(effectiveInput);
+            setErrorAttributes(span, message);
+            recordBaseMetrics(duration, effectiveInput.provider, effectiveInput.model, attrs);
+            recordError(effectiveInput.provider, effectiveInput.model, attrs);
+            // If this was a budget block, record the metric
+            if (error instanceof Error &&
+                error.name === "ToadBudgetExceededError") {
+                recordBudgetBlocked("daily");
+            }
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    });
+}

package/dist/core/tracer.d.ts

@@ -0,0 +1,9 @@
+import type { ToadEyeConfig } from "../types/index.js";
+import { BudgetTracker } from "../budget/index.js";
+import "../instrumentations/openai.js";
+import "../instrumentations/anthropic.js";
+import "../instrumentations/gemini.js";
+export declare function getConfig(): ToadEyeConfig | null;
+export declare function getBudgetTracker(): BudgetTracker | null;
+export declare function initObservability(config: ToadEyeConfig): void;
+export declare function shutdown(): Promise<void>;

package/dist/core/tracer.js

@@ -0,0 +1,93 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
+import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { initMetrics } from "./metrics.js";
+import { enableAll, disableAll } from "../instrumentations/registry.js";
+import { BudgetTracker } from "../budget/index.js";
+// Side-effect imports: register provider instrumentations
+import "../instrumentations/openai.js";
+import "../instrumentations/anthropic.js";
+import "../instrumentations/gemini.js";
+const DEFAULT_ENDPOINT = "http://localhost:4318";
+const DEFAULT_CLOUD_ENDPOINT = "https://cloud.toad-eye.dev";
+let sdk = null;
+let currentConfig = null;
+let budgetTracker = null;
+export function getConfig() {
+    return currentConfig;
+}
+export function getBudgetTracker() {
+    return budgetTracker;
+}
+function validateConfig(config) {
+    if (!config.serviceName?.trim()) {
+        throw new Error("toad-eye: serviceName is required and cannot be empty");
+    }
+    if (config.endpoint !== undefined && !config.endpoint.startsWith("http")) {
+        throw new Error(`toad-eye: endpoint must be a valid URL, got "${config.endpoint}"`);
+    }
+    if (config.apiKey !== undefined && !config.apiKey.startsWith("toad_")) {
+        throw new Error('toad-eye: apiKey must start with "toad_" prefix');
+    }
+}
+/** Resolve endpoint and auth headers based on config mode (self-hosted vs cloud). */
+function resolveTransport(config) {
+    const isCloudMode = config.apiKey !== undefined;
+    const endpoint = isCloudMode
+        ? (config.cloudEndpoint ?? DEFAULT_CLOUD_ENDPOINT)
+        : (config.endpoint ?? DEFAULT_ENDPOINT);
+    const headers = isCloudMode
+        ? { Authorization: `Bearer ${config.apiKey}` }
+        : {};
+    return { endpoint, headers, isCloudMode };
+}
+export function initObservability(config) {
+    if (sdk)
+        return;
+    validateConfig(config);
+    const { endpoint, headers, isCloudMode } = resolveTransport(config);
+    const resource = resourceFromAttributes({
+        [ATTR_SERVICE_NAME]: config.serviceName,
+    });
+    const traceExporter = new OTLPTraceExporter({
+        url: `${endpoint}/v1/traces`,
+        headers,
+    });
+    const metricExporter = new OTLPMetricExporter({
+        url: `${endpoint}/v1/metrics`,
+        headers,
+    });
+    const metricReader = new PeriodicExportingMetricReader({
+        exporter: metricExporter,
+        exportIntervalMillis: isCloudMode ? 10_000 : 5_000,
+    });
+    sdk = new NodeSDK({
+        resource,
+        traceExporter,
+        metricReader,
+    });
+    sdk.start();
+    currentConfig = config;
+    initMetrics();
+    if (isCloudMode) {
+        console.log(`toad-eye: cloud mode enabled → sending telemetry to ${endpoint}`);
+    }
+    if (config.budgets) {
+        budgetTracker = new BudgetTracker(config.budgets, config.onBudgetExceeded ?? "warn", config.downgradeCallback);
+    }
+    if (config.instrument?.length) {
+        enableAll(config.instrument);
+    }
+}
+export async function shutdown() {
+    if (!sdk)
+        return;
+    disableAll();
+    await sdk.shutdown();
+    sdk = null;
+    currentConfig = null;
+    budgetTracker = null;
+}

package/dist/drift/baseline.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Baseline storage — save and load embedding baselines as JSON files.
+ *
+ * Baseline files contain embeddings of "known good" responses.
+ * Drift monitoring compares current responses against these baselines.
+ */
+export interface DriftBaseline {
+    readonly model: string;
+    readonly provider: string;
+    readonly embeddingModel: string;
+    readonly embeddings: readonly (readonly number[])[];
+    readonly sampleCount: number;
+    readonly createdAt: string;
+}
+export declare function saveBaseline(filepath: string, baseline: DriftBaseline): void;
+export declare function loadBaseline(filepath: string): DriftBaseline | undefined;

package/dist/drift/baseline.js

@@ -0,0 +1,16 @@
+/**
+ * Baseline storage — save and load embedding baselines as JSON files.
+ *
+ * Baseline files contain embeddings of "known good" responses.
+ * Drift monitoring compares current responses against these baselines.
+ */
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+export function saveBaseline(filepath, baseline) {
+    writeFileSync(filepath, JSON.stringify(baseline, null, 2), "utf-8");
+}
+export function loadBaseline(filepath) {
+    if (!existsSync(filepath))
+        return undefined;
+    const raw = readFileSync(filepath, "utf-8");
+    return JSON.parse(raw);
+}

package/dist/drift/cosine.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Cosine similarity and drift calculation for embedding vectors.
+ *
+ * drift = 1 - cosine_similarity(current, baseline)
+ * - drift = 0 → identical to baseline
+ * - drift > 0.3 → significant deviation (default alert threshold)
+ */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/**
+ * Calculate drift as distance from the nearest baseline embedding.
+ * Returns the minimum drift value (closest match).
+ */
+export declare function cosineDrift(current: readonly number[], baselines: readonly (readonly number[])[]): number;

package/dist/drift/cosine.js

@@ -0,0 +1,39 @@
+/**
+ * Cosine similarity and drift calculation for embedding vectors.
+ *
+ * drift = 1 - cosine_similarity(current, baseline)
+ * - drift = 0 → identical to baseline
+ * - drift > 0.3 → significant deviation (default alert threshold)
+ */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        const ai = a[i];
+        const bi = b[i];
+        dot += ai * bi;
+        normA += ai * ai;
+        normB += bi * bi;
+    }
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    if (denominator === 0)
+        return 0;
+    return dot / denominator;
+}
+/**
+ * Calculate drift as distance from the nearest baseline embedding.
+ * Returns the minimum drift value (closest match).
+ */
+export function cosineDrift(current, baselines) {
+    if (baselines.length === 0)
+        return 1;
+    let maxSimilarity = -1;
+    for (const baseline of baselines) {
+        const similarity = cosineSimilarity(current, baseline);
+        if (similarity > maxSimilarity) {
+            maxSimilarity = similarity;
+        }
+    }
+    return 1 - maxSimilarity;
+}

package/dist/drift/embedding-provider.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Embedding provider abstraction for semantic drift monitoring.
+ *
+ * Implementations convert text into float vectors for cosine similarity comparison.
+ * Default: OpenAI text-embedding-3-small (cheap, fast).
+ * Users can provide a custom implementation via the EmbeddingProvider interface.
+ */
+export interface EmbeddingProvider {
+    readonly name: string;
+    embed(text: string): Promise<readonly number[]>;
+}
+export interface OpenAIEmbeddingConfig {
+    readonly provider: "openai";
+    readonly apiKey: string;
+    readonly model?: string | undefined;
+}
+export interface CustomEmbeddingConfig {
+    readonly provider: "custom";
+    readonly embed: (text: string) => Promise<readonly number[]>;
+}
+export type EmbeddingConfig = OpenAIEmbeddingConfig | CustomEmbeddingConfig;

package/dist/drift/embedding-provider.js

@@ -0,0 +1,8 @@
+/**
+ * Embedding provider abstraction for semantic drift monitoring.
+ *
+ * Implementations convert text into float vectors for cosine similarity comparison.
+ * Default: OpenAI text-embedding-3-small (cheap, fast).
+ * Users can provide a custom implementation via the EmbeddingProvider interface.
+ */
+export {};

package/dist/drift/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Semantic Drift Monitoring — detect silent LLM quality degradation
+ * by comparing current responses to a saved baseline via embeddings.
+ */
+export type { EmbeddingProvider, EmbeddingConfig, OpenAIEmbeddingConfig, CustomEmbeddingConfig, } from "./embedding-provider.js";
+export { createOpenAIEmbeddingProvider } from "./openai-embeddings.js";
+export { cosineSimilarity, cosineDrift } from "./cosine.js";
+export { saveBaseline, loadBaseline, type DriftBaseline } from "./baseline.js";
+export { createDriftMonitor, type DriftMonitor, type DriftMonitorConfig, } from "./monitor.js";

package/dist/drift/index.js

@@ -0,0 +1,8 @@
+/**
+ * Semantic Drift Monitoring — detect silent LLM quality degradation
+ * by comparing current responses to a saved baseline via embeddings.
+ */
+export { createOpenAIEmbeddingProvider } from "./openai-embeddings.js";
+export { cosineSimilarity, cosineDrift } from "./cosine.js";
+export { saveBaseline, loadBaseline } from "./baseline.js";
+export { createDriftMonitor, } from "./monitor.js";

package/dist/drift/monitor.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Drift monitor — runtime semantic drift detection with sampling.
+ *
+ * Not every response is checked (embedding API calls cost money).
+ * By default, every 10th response is sampled. The drift value is
+ * recorded as a Prometheus histogram metric with provider/model labels.
+ */
+import type { EmbeddingConfig } from "./embedding-provider.js";
+export interface DriftMonitorConfig {
+    readonly embedding: EmbeddingConfig;
+    readonly baselinePath: string;
+    readonly sampleRate?: number | undefined;
+}
+export interface DriftMonitor {
+    /**
+     * Check a response for semantic drift. Respects sampling rate —
+     * only every Nth call actually computes embeddings.
+     * Returns the drift value if checked, undefined if skipped.
+     */
+    check(response: string, provider: string, model: string): Promise<number | undefined>;
+    /**
+     * Fire-and-forget drift check — does NOT block the caller.
+     * Errors are logged to console.warn, never thrown.
+     * Use this in hot paths (e.g. inside traceLLMCall) to avoid
+     * blocking the LLM response delivery.
+     */
+    checkInBackground(response: string, provider: string, model: string): void;
+}
+export declare function createDriftMonitor(config: DriftMonitorConfig): DriftMonitor;