toad-eye 2.2.0 → 2.3.0

package/dist/alerts/channels.js

@@ -1,4 +1,10 @@
-import nodemailer from "nodemailer";
+// Cache nodemailer transporters by connection identity to avoid creating a new
+// SMTP connection for every alert (frequent alerts would exhaust connection pools).
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const smtpTransporterCache = new Map();
+function smtpCacheKey(config) {
+    return `${config.host}:${config.port}:${config.user}`;
+}
 function formatMessage(alert) {
     const topModelsText = alert.topModels.length > 0
         ? `\nTop models by spend:\n${alert.topModels.map((m) => `  • ${m.model}: $${m.value.toFixed(4)}`).join("\n")}`
@@ -52,11 +58,23 @@ async function sendWebhook(config, alert) {
         throw new Error(`Webhook delivery failed: ${res.status}`);
 }
 async function sendEmail(config, alert) {
-    const transporter = nodemailer.createTransport({
-        host: config.host,
-        port: config.port,
-        auth: { user: config.user, pass: config.password },
-    });
+    const key = smtpCacheKey(config);
+    let transporter = smtpTransporterCache.get(key);
+    if (!transporter) {
+        let createTransport;
+        try {
+            ({ createTransport } = await import("nodemailer"));
+        }
+        catch {
+            throw new Error("toad-eye: email alerts require nodemailer — install it: npm install nodemailer");
+        }
+        transporter = createTransport({
+            host: config.host,
+            port: config.port,
+            auth: { user: config.user, pass: config.password },
+        });
+        smtpTransporterCache.set(key, transporter);
+    }
     const to = typeof config.to === "string"
         ? config.to
         : Array.from(config.to).join(", ");

package/dist/alerts/index.js

@@ -3,10 +3,54 @@ export { parseCondition } from "./conditions.js";
 import { readFileSync } from "node:fs";
 import yaml from "yaml";
 import { AlertManager } from "./manager.js";
+/** Resolve ${ENV_VAR} references in string values throughout the config. */
+function resolveEnvVars(value) {
+    return value.replace(/\$\{([^}]+)\}/g, (_, name) => {
+        return process.env[name] ?? `\${${name}}`;
+    });
+}
+function interpolateConfig(obj) {
+    if (typeof obj === "string")
+        return resolveEnvVars(obj);
+    if (Array.isArray(obj))
+        return obj.map(interpolateConfig);
+    if (obj !== null && typeof obj === "object") {
+        return Object.fromEntries(Object.entries(obj).map(([k, v]) => [
+            k,
+            interpolateConfig(v),
+        ]));
+    }
+    return obj;
+}
+function validateConfig(config) {
+    if (typeof config !== "object" || config === null) {
+        throw new Error("toad-eye: alerts config must be an object");
+    }
+    const c = config;
+    if (!Array.isArray(c["alerts"])) {
+        throw new Error('toad-eye: alerts config missing required "alerts" array');
+    }
+    for (const rule of c["alerts"]) {
+        if (typeof rule !== "object" || rule === null) {
+            throw new Error("toad-eye: each alert rule must be an object");
+        }
+        const r = rule;
+        for (const field of ["name", "metric", "condition"]) {
+            if (typeof r[field] !== "string" || !r[field]) {
+                throw new Error(`toad-eye: alert rule missing required string field "${field}"`);
+            }
+        }
+        if (!Array.isArray(r["channels"]) || r["channels"].length === 0) {
+            throw new Error(`toad-eye: alert rule "${String(r["name"])}" must have at least one channel`);
+        }
+    }
+}
 export function startAlertsFromFile(configPath) {
     const raw = readFileSync(configPath, "utf-8");
-    const config = yaml.parse(raw);
-    const manager = new AlertManager(config);
+    const parsed = yaml.parse(raw);
+    const interpolated = interpolateConfig(parsed);
+    validateConfig(interpolated);
+    const manager = new AlertManager(interpolated);
     manager.start();
     return manager;
 }

package/dist/alerts/manager.d.ts

@@ -8,6 +8,7 @@ export declare class AlertManager {
     stop(): void;
     private isInCooldown;
     private evaluate;
+    private evaluateRule;
     private fireChannels;
     private postAnnotation;
 }

package/dist/alerts/manager.js

@@ -5,6 +5,7 @@ const DEFAULT_PROMETHEUS_URL = "http://localhost:9090";
 const DEFAULT_GRAFANA_URL = "http://localhost:3100";
 const DEFAULT_EVAL_INTERVAL_SECONDS = 60;
 const DEFAULT_COOLDOWN_MINUTES = 30;
+const MIN_EVAL_INTERVAL_SECONDS = 10;
 export class AlertManager {
     config;
     cooldowns = new Map();
@@ -13,10 +14,15 @@ export class AlertManager {
         this.config = config;
     }
     start() {
-        const intervalMs = (this.config.evalIntervalSeconds ?? DEFAULT_EVAL_INTERVAL_SECONDS) * 1000;
+        const configured = this.config.evalIntervalSeconds ?? DEFAULT_EVAL_INTERVAL_SECONDS;
+        const clampedSeconds = Math.max(configured, MIN_EVAL_INTERVAL_SECONDS);
+        if (configured < MIN_EVAL_INTERVAL_SECONDS) {
+            console.warn(`[toad-eye alerts] evalIntervalSeconds ${configured} is below minimum ${MIN_EVAL_INTERVAL_SECONDS}s — using ${clampedSeconds}s`);
+        }
+        const intervalMs = clampedSeconds * 1000;
         this.intervalId = setInterval(() => void this.evaluate(), intervalMs);
         void this.evaluate();
-        console.log(`[toad-eye alerts] Started — ${this.config.alerts.length} rule(s), eval every ${intervalMs / 1000}s`);
+        console.log(`[toad-eye alerts] Started — ${this.config.alerts.length} rule(s), eval every ${clampedSeconds}s`);
     }
     stop() {
         if (this.intervalId !== undefined) {
@@ -38,26 +44,35 @@ export class AlertManager {
     async evaluate() {
         const prometheusUrl = this.config.prometheusUrl ?? DEFAULT_PROMETHEUS_URL;
         const grafanaUrl = this.config.grafanaUrl ?? DEFAULT_GRAFANA_URL;
-        for (const rule of this.config.alerts) {
-            if (this.isInCooldown(rule))
-                continue;
-            const result = await evaluateCondition(prometheusUrl, rule);
-            if (!result?.triggered)
-                continue;
-            this.cooldowns.set(rule.name, Date.now());
-            const firedAlert = {
-                rule,
-                value: result.value,
-                threshold: result.threshold,
-                topModels: result.topModels,
-                firedAt: new Date(),
-            };
-            console.log(`[toad-eye alerts] 🚨 "${rule.name}" fired — value=${result.value.toFixed(4)}, threshold=${result.threshold}`);
-            await Promise.allSettled([
-                this.fireChannels(firedAlert),
-                this.postAnnotation(grafanaUrl, firedAlert.rule.name, firedAlert.value, firedAlert.threshold),
-            ]);
+        // Evaluate all rules in parallel — each rule is independent
+        await Promise.allSettled(this.config.alerts.map((rule) => this.evaluateRule(prometheusUrl, grafanaUrl, rule)));
+    }
+    async evaluateRule(prometheusUrl, grafanaUrl, rule) {
+        if (this.isInCooldown(rule))
+            return;
+        let result;
+        try {
+            result = await evaluateCondition(prometheusUrl, rule);
+        }
+        catch (err) {
+            console.error(`[toad-eye alerts] Failed to evaluate rule "${rule.name}":`, err);
+            return;
         }
+        if (!result?.triggered)
+            return;
+        this.cooldowns.set(rule.name, Date.now());
+        const firedAlert = {
+            rule,
+            value: result.value,
+            threshold: result.threshold,
+            topModels: result.topModels,
+            firedAt: new Date(),
+        };
+        console.log(`[toad-eye alerts] 🚨 "${rule.name}" fired — value=${result.value.toFixed(4)}, threshold=${result.threshold}`);
+        await Promise.allSettled([
+            this.fireChannels(firedAlert),
+            this.postAnnotation(grafanaUrl, firedAlert.rule.name, firedAlert.value, firedAlert.threshold),
+        ]);
     }
     async fireChannels(alert) {
         const channels = this.config.channels ?? {};

package/dist/budget/tracker.d.ts

@@ -9,18 +9,28 @@ export declare class BudgetTracker {
     private resetIfNewDay;
     /**
      * Check budget BEFORE making an LLM call.
-     * Returns modified provider/model if downgrade mode triggers,
-     * throws ToadBudgetExceededError if block mode triggers,
-     * returns null if no action needed.
+     * - block mode: throws ToadBudgetExceededError if budget is exceeded.
+     * - downgrade mode: returns modified provider/model + budget type that triggered it.
+     * - warn mode: returns null (warning emitted after call via recordCost).
+     *
+     * When estimatedCost is provided, it is added to reservedCost so concurrent
+     * in-flight requests are counted against the budget before their actual cost is known.
+     * Call recordCost with the same estimatedCost to release the reservation.
      */
-    checkBefore(provider: string, model: string, userId?: string): {
+    checkBefore(provider: string, model: string, userId?: string, estimatedCost?: number): {
         provider: string;
         model: string;
+        budget: BudgetExceededInfo["budget"];
     } | null;
-    /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
-    recordCost(cost: number, model: string, userId?: string): BudgetExceededInfo | null;
+    /**
+     * Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed.
+     * Pass the same estimatedCost used in checkBefore() to release the reservation atomically.
+     */
+    recordCost(cost: number, model: string, userId?: string, reservedAmount?: number): BudgetExceededInfo | null;
     /** Find which budget (if any) is exceeded. */
     private findExceeded;
+    /** Release a cost reservation when an LLM call fails (no cost was incurred). */
+    releaseReservation(reservedAmount: number): void;
     /** Get current budget usage as percentage (0-100+). */
     getUsagePercent(): number;
     /** Get current state snapshot (for persistence or debugging). */

package/dist/budget/tracker.js

@@ -14,6 +14,7 @@ export class BudgetTracker {
         this.state = {
             date: todayUTC(),
             totalCost: 0,
+            reservedCost: 0,
             perUser: new Map(),
             perModel: new Map(),
         };
@@ -25,6 +26,7 @@ export class BudgetTracker {
             this.state = {
                 date: today,
                 totalCost: 0,
+                reservedCost: 0,
                 perUser: new Map(),
                 perModel: new Map(),
             };
@@ -32,27 +34,40 @@ export class BudgetTracker {
     }
     /**
      * Check budget BEFORE making an LLM call.
-     * Returns modified provider/model if downgrade mode triggers,
-     * throws ToadBudgetExceededError if block mode triggers,
-     * returns null if no action needed.
+     * - block mode: throws ToadBudgetExceededError if budget is exceeded.
+     * - downgrade mode: returns modified provider/model + budget type that triggered it.
+     * - warn mode: returns null (warning emitted after call via recordCost).
+     *
+     * When estimatedCost is provided, it is added to reservedCost so concurrent
+     * in-flight requests are counted against the budget before their actual cost is known.
+     * Call recordCost with the same estimatedCost to release the reservation.
      */
-    checkBefore(provider, model, userId) {
+    checkBefore(provider, model, userId, estimatedCost = 0) {
         this.resetIfNewDay();
-        const exceeded = this.findExceeded(model, userId);
-        if (!exceeded)
-            return null;
-        if (this.mode === "block") {
-            throw new ToadBudgetExceededError(exceeded);
-        }
-        if (this.mode === "downgrade" && this.downgrade) {
-            return this.downgrade({ provider, model });
+        const exceeded = this.findExceeded(model, userId, estimatedCost);
+        if (exceeded) {
+            if (this.mode === "block") {
+                throw new ToadBudgetExceededError(exceeded);
+            }
+            if (this.mode === "downgrade" && this.downgrade) {
+                this.state.reservedCost += estimatedCost;
+                return {
+                    ...this.downgrade({ provider, model }),
+                    budget: exceeded.budget,
+                };
+            }
         }
-        // 'warn' mode — continue, warning emitted after call
+        // warn mode or no budget exceeded: reserve estimated cost and proceed
+        this.state.reservedCost += estimatedCost;
         return null;
     }
-    /** Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed. */
-    recordCost(cost, model, userId) {
+    /**
+     * Record cost AFTER a successful LLM call. Returns exceeded info if budget was just crossed.
+     * Pass the same estimatedCost used in checkBefore() to release the reservation atomically.
+     */
+    recordCost(cost, model, userId, reservedAmount = 0) {
         this.resetIfNewDay();
+        this.state.reservedCost = Math.max(0, this.state.reservedCost - reservedAmount);
         this.state.totalCost += cost;
         this.state.perModel.set(model, (this.state.perModel.get(model) ?? 0) + cost);
         if (userId) {
@@ -61,9 +76,10 @@ export class BudgetTracker {
         return this.findExceeded(model, userId);
     }
     /** Find which budget (if any) is exceeded. */
-    findExceeded(model, userId) {
+    findExceeded(model, userId, additionalCost = 0) {
         if (this.config.daily !== undefined &&
-            this.state.totalCost >= this.config.daily) {
+            this.state.totalCost + this.state.reservedCost + additionalCost >=
+                this.config.daily) {
             return {
                 budget: "daily",
                 limit: this.config.daily,
@@ -95,6 +111,10 @@ export class BudgetTracker {
         }
         return null;
     }
+    /** Release a cost reservation when an LLM call fails (no cost was incurred). */
+    releaseReservation(reservedAmount) {
+        this.state.reservedCost = Math.max(0, this.state.reservedCost - reservedAmount);
+    }
     /** Get current budget usage as percentage (0-100+). */
     getUsagePercent() {
         if (this.config.daily === undefined || this.config.daily === 0)
@@ -114,6 +134,7 @@ export class BudgetTracker {
         this.state = {
             date: saved.date,
             totalCost: saved.totalCost,
+            reservedCost: 0,
             perUser: new Map(Object.entries(saved.perUser)),
             perModel: new Map(Object.entries(saved.perModel)),
         };

package/dist/budget/types.d.ts

@@ -21,6 +21,8 @@ export type DowngradeCallback = (original: {
 export interface BudgetState {
     date: string;
     totalCost: number;
+    /** Sum of estimated costs for in-flight requests (released when recordCost is called). */
+    reservedCost: number;
     perUser: Map<string, number>;
     perModel: Map<string, number>;
 }

package/dist/cli.d.ts

@@ -1,2 +1,9 @@
 #!/usr/bin/env node
-export {};
+/** Find the docker container matching a service by exact service name. Exported for testing. */
+export declare function findContainerByService(containers: ReadonlyArray<{
+    Service: string;
+    State: string;
+}>, serviceKey: string): {
+    Service: string;
+    State: string;
+} | undefined;

package/dist/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { execSync, spawn } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
 import { cpSync, existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -12,10 +12,19 @@ const DEMO_TICK_INTERVAL_MS = 2000;
 const DEMO_TOKEN_INPUT_RANGE = [50, 500];
 const DEMO_TOKEN_OUTPUT_RANGE = [20, 300];
 const SERVICES = [
-    { name: "Grafana", url: "http://localhost:3100", login: "admin / admin" },
-    { name: "Jaeger UI", url: "http://localhost:16686" },
-    { name: "Prometheus", url: "http://localhost:9090" },
-    { name: "OTel Collector", url: "http://localhost:4318" },
+    {
+        name: "Grafana",
+        service: "grafana",
+        url: "http://localhost:3100",
+        login: "admin / admin",
+    },
+    { name: "Jaeger UI", service: "jaeger", url: "http://localhost:16686" },
+    { name: "Prometheus", service: "prometheus", url: "http://localhost:9090" },
+    {
+        name: "OTel Collector",
+        service: "collector",
+        url: "http://localhost:4318",
+    },
 ];
 function getTemplatesDir() {
     const thisFile = fileURLToPath(import.meta.url);
@@ -35,8 +44,9 @@ function requireInfra(message = `Run \`npx toad-eye init\` first.`) {
 }
 function init() {
     const dest = join(process.cwd(), INFRA_DIR);
-    if (existsSync(dest)) {
-        console.log(`\u2705 ${INFRA_DIR}/ already exists, skipping.`);
+    const force = process.argv.includes("--force");
+    if (existsSync(dest) && !force) {
+        console.log(`✅ ${INFRA_DIR}/ already exists, skipping. Use --force to overwrite.`);
         return;
     }
     const templatesDir = getTemplatesDir();
@@ -44,39 +54,59 @@ function init() {
         console.error(`\u274c Templates not found at ${templatesDir}. Reinstall toad-eye.`);
         process.exit(1);
     }
+    const existed = existsSync(dest);
     mkdirSync(dest, { recursive: true });
     cpSync(templatesDir, dest, { recursive: true });
-    console.log(`\u2705 Created ${INFRA_DIR}/ with observability stack config.`);
+    const action = force && existed ? "Updated" : "Created";
+    console.log(`✅ ${action} ${INFRA_DIR}/ with observability stack config.`);
     console.log();
     console.log("Next steps:");
     console.log("  npx toad-eye up      Start the stack");
     console.log("  npx toad-eye status  Check running services");
 }
+function requireDocker() {
+    try {
+        execFileSync("docker", ["--version"], { stdio: "ignore" });
+    }
+    catch {
+        console.error("❌ Docker not found. Install Docker Desktop: https://docs.docker.com/get-started/get-docker/");
+        process.exit(1);
+    }
+}
 function up() {
+    requireDocker();
     const composeFile = requireInfra();
-    console.log("\u{1f438} Starting observability stack...");
-    execSync(`docker compose -f ${composeFile} up -d`, { stdio: "inherit" });
+    console.log("🐸 Starting observability stack...");
+    console.log("   (first run downloads ~500MB of Docker images — this may take a few minutes)\n");
+    execFileSync("docker", ["compose", "-f", composeFile, "up", "-d"], {
+        stdio: "inherit",
+    });
     console.log();
     status();
+    console.log("   Run 'npx toad-eye demo' to send test data and verify the stack works.");
 }
 function down() {
     const composeFile = requireInfra("Nothing to stop.");
     console.log("\u{1f44b} Stopping observability stack...");
-    execSync(`docker compose -f ${composeFile} down`, { stdio: "inherit" });
+    execFileSync("docker", ["compose", "-f", composeFile, "down"], {
+        stdio: "inherit",
+    });
     console.log("\u2705 Stack stopped.");
 }
+/** Find the docker container matching a service by exact service name. Exported for testing. */
+export function findContainerByService(containers, serviceKey) {
+    return containers.find((c) => c.Service === serviceKey);
+}
 function status() {
     const composeFile = requireInfra();
     try {
-        const output = execSync(`docker compose -f ${composeFile} ps --format json`, {
-            encoding: "utf-8",
-        });
+        const output = execFileSync("docker", ["compose", "-f", composeFile, "ps", "--format", "json"], { encoding: "utf-8" });
         const lines = output.trim().split("\n").filter(Boolean);
         const containers = lines.map((line) => JSON.parse(line));
         console.log("\u{1f438} toad-eye stack:");
         console.log();
         for (const svc of SERVICES) {
-            const container = containers.find((c) => svc.name.toLowerCase().includes(c.Service.toLowerCase()));
+            const container = findContainerByService(containers, svc.service);
             const state = container?.State === "running" ? "\u{1f7e2}" : "\u{1f534}";
             const loginInfo = "login" in svc ? ` (${svc.login})` : "";
             console.log(`  ${state} ${svc.name.padEnd(16)} ${svc.url}${loginInfo}`);
@@ -121,12 +151,25 @@ async function simulateLLMCall(provider, model, prompt) {
 }
 async function demo() {
     requireInfra();
+    // Warn early if OTel Collector is unreachable — data would be silently lost otherwise
+    try {
+        await fetch("http://localhost:4318/v1/traces", {
+            method: "POST",
+            body: "[]",
+            signal: AbortSignal.timeout(2000),
+        });
+    }
+    catch {
+        console.warn("⚠️  OTel Collector not reachable at localhost:4318 — data will not be recorded.\n   Run 'npx toad-eye up' first, then retry.\n");
+    }
     initObservability({
         serviceName: "toad-eye-demo",
         endpoint: "http://localhost:4318",
     });
-    console.log("\u{1f438}\u{1f441}\u{fe0f} toad-eye demo — sending mock LLM traffic");
+    console.log("🐸👁️ toad-eye demo — sending mock LLM traffic");
     console.log("    Press Ctrl+C to stop\n");
+    console.log("✅ Stack verified! Add this to your app:\n");
+    console.log("   import { initObservability } from 'toad-eye';\n   initObservability({ serviceName: 'my-app', instrument: ['openai'] });\n");
     process.on("SIGINT", async () => {
         console.log("\n\u{1f44b} Shutting down...");
         await shutdown();
@@ -186,47 +229,56 @@ async function exportTraceCommand() {
     }
 }
 function help() {
+    const cmd = (name) => name.padEnd(15);
     console.log(`
-\u{1f438} toad-eye CLI — observability stack for LLM services
+🐸 toad-eye CLI — observability stack for LLM services
 
 Commands:
-  init     Copy observability configs into your project
-  up       Start the stack (OTel Collector + Prometheus + Jaeger + Grafana)
-  down     Stop the stack
-  status   Show running services and URLs
-  demo           Send mock LLM traffic to see data in Grafana
-  export-trace   Export a Jaeger trace to toad-eval YAML
-  help           Show this message
+  ${cmd("init")}Copy observability configs into your project
+  ${cmd("init --force")}Overwrite existing configs with latest templates
+  ${cmd("up")}Start the stack (OTel Collector + Prometheus + Jaeger + Grafana)
+  ${cmd("down")}Stop the stack
+  ${cmd("status")}Show running services and URLs
+  ${cmd("demo")}Send mock LLM traffic to see data in Grafana
+  ${cmd("export-trace")}Export a Jaeger trace to toad-eval YAML
+  ${cmd("help")}Show this message
 `);
 }
-const command = process.argv[2];
-switch (command) {
-    case "init":
-        init();
-        break;
-    case "up":
-        up();
-        break;
-    case "down":
-        down();
-        break;
-    case "status":
-        status();
-        break;
-    case "demo":
-        demo();
-        break;
-    case "export-trace":
-        exportTraceCommand();
-        break;
-    case "help":
-    case "--help":
-    case "-h":
-    case undefined:
-        help();
-        break;
-    default:
-        console.error(`Unknown command: ${command}`);
-        help();
-        process.exit(1);
+function runCli() {
+    const command = process.argv[2];
+    switch (command) {
+        case "init":
+            init();
+            break;
+        case "up":
+            up();
+            break;
+        case "down":
+            down();
+            break;
+        case "status":
+            status();
+            break;
+        case "demo":
+            demo();
+            break;
+        case "export-trace":
+            exportTraceCommand();
+            break;
+        case "help":
+        case "--help":
+        case "-h":
+        case undefined:
+            help();
+            break;
+        default:
+            console.error(`Unknown command: ${command}`);
+            help();
+            process.exit(1);
+    }
+}
+// Only auto-execute when run as CLI entry point, not when imported (e.g., in tests)
+if (process.argv[1] !== undefined &&
+    fileURLToPath(import.meta.url) === resolve(process.argv[1])) {
+    runCli();
 }

package/dist/core/metrics.d.ts

@@ -1,4 +1,6 @@
 export declare function initMetrics(): void;
+/** Reset metrics state so the next initMetrics() creates fresh instruments. Called from shutdown(). */
+export declare function resetMetrics(): void;
 export declare function recordRequestDuration(ms: number, provider: string, model: string, attrs?: Record<string, string>): void;
 export declare function recordRequestCost(usd: number, provider: string, model: string, attrs?: Record<string, string>): void;
 export declare function recordTokens(count: number, provider: string, model: string, attrs?: Record<string, string>): void;
@@ -13,3 +15,5 @@ export declare function recordTimeToFirstToken(ms: number, provider: string, mod
 export declare function recordBudgetExceeded(budgetType: string): void;
 export declare function recordBudgetBlocked(budgetType: string): void;
 export declare function recordBudgetDowngraded(budgetType: string): void;
+export declare function recordResponseEmpty(provider: string, model: string, attrs?: Record<string, string>): void;
+export declare function recordResponseLatencyPerToken(msPerToken: number, provider: string, model: string, attrs?: Record<string, string>): void;

package/dist/core/metrics.js

@@ -14,6 +14,8 @@ let timeToFirstToken;
 let budgetExceeded;
 let budgetBlocked;
 let budgetDowngraded;
+let responseEmpty;
+let responseLatencyPerToken;
 let initialized = false;
 export function initMetrics() {
     if (initialized)
@@ -64,8 +66,19 @@ export function initMetrics() {
     budgetDowngraded = meter.createCounter(GEN_AI_METRICS.BUDGET_DOWNGRADED, {
         description: "Number of LLM calls downgraded due to budget limits",
     });
+    responseEmpty = meter.createCounter(GEN_AI_METRICS.RESPONSE_EMPTY, {
+        description: "Number of LLM responses with empty or whitespace-only completion",
+    });
+    responseLatencyPerToken = meter.createHistogram(GEN_AI_METRICS.RESPONSE_LATENCY_PER_TOKEN, {
+        description: "Generation latency normalized per output token (ms/token). Higher values indicate slower generation speed.",
+        unit: "ms",
+    });
     initialized = true;
 }
+/** Reset metrics state so the next initMetrics() creates fresh instruments. Called from shutdown(). */
+export function resetMetrics() {
+    initialized = false;
+}
 /** Build metric labels: provider + model + optional FinOps attributes. */
 function baseLabels(provider, model, attrs) {
     return {
@@ -128,3 +141,9 @@ export function recordBudgetBlocked(budgetType) {
 export function recordBudgetDowngraded(budgetType) {
     budgetDowngraded.add(1, { budget_type: budgetType });
 }
+export function recordResponseEmpty(provider, model, attrs) {
+    responseEmpty.add(1, baseLabels(provider, model, attrs));
+}
+export function recordResponseLatencyPerToken(msPerToken, provider, model, attrs) {
+    responseLatencyPerToken.record(msPerToken, baseLabels(provider, model, attrs));
+}

package/dist/core/pricing.d.ts

@@ -12,6 +12,8 @@ export interface ModelPricing {
  * Useful for enterprise contracts or custom/fine-tuned models.
  */
 export declare function setCustomPricing(pricing: Record<string, ModelPricing>): void;
+/** Reset custom pricing to empty. Called from shutdown() to avoid stale state across re-inits. */
+export declare function resetCustomPricing(): void;
 /**
  * Get pricing for a model. Custom pricing takes precedence.
  * Returns undefined if model is not in any pricing table.

package/dist/core/pricing.js

@@ -31,6 +31,10 @@ let customPricing = {};
 export function setCustomPricing(pricing) {
     customPricing = { ...pricing };
 }
+/** Reset custom pricing to empty. Called from shutdown() to avoid stale state across re-inits. */
+export function resetCustomPricing() {
+    customPricing = {};
+}
 /**
  * Get pricing for a model. Custom pricing takes precedence.
  * Returns undefined if model is not in any pricing table.