toad-eye 2.1.1 → 2.2.0

package/dist/core/metrics.d.ts

@@ -9,6 +9,7 @@ export declare function recordAgentToolUsage(toolName: string): void;
 export declare function recordGuardEvaluation(ruleName: string): void;
 export declare function recordGuardWouldBlock(ruleName: string): void;
 export declare function recordSemanticDrift(drift: number, provider: string, model: string): void;
+export declare function recordTimeToFirstToken(ms: number, provider: string, model: string): void;
 export declare function recordBudgetExceeded(budgetType: string): void;
 export declare function recordBudgetBlocked(budgetType: string): void;
 export declare function recordBudgetDowngraded(budgetType: string): void;

package/dist/core/metrics.js

@@ -10,6 +10,7 @@ let agentToolUsage;
 let guardEvaluations;
 let guardWouldBlock;
 let semanticDrift;
+let timeToFirstToken;
 let budgetExceeded;
 let budgetBlocked;
 let budgetDowngraded;
@@ -50,6 +51,10 @@ export function initMetrics() {
     semanticDrift = meter.createHistogram(GEN_AI_METRICS.SEMANTIC_DRIFT, {
         description: "Semantic drift from baseline (0 = identical, 1 = completely different)",
     });
+    timeToFirstToken = meter.createHistogram(GEN_AI_METRICS.TIME_TO_FIRST_TOKEN, {
+        description: "Time from request start to first streaming token (ms)",
+        unit: "ms",
+    });
     budgetExceeded = meter.createCounter(GEN_AI_METRICS.BUDGET_EXCEEDED, {
         description: "Number of times a budget limit was exceeded",
     });
@@ -108,6 +113,12 @@ export function recordSemanticDrift(drift, provider, model) {
         [GEN_AI_ATTRS.REQUEST_MODEL]: model,
     });
 }
+export function recordTimeToFirstToken(ms, provider, model) {
+    timeToFirstToken.record(ms, {
+        [GEN_AI_ATTRS.PROVIDER]: provider,
+        [GEN_AI_ATTRS.REQUEST_MODEL]: model,
+    });
+}
 export function recordBudgetExceeded(budgetType) {
     budgetExceeded.add(1, { budget_type: budgetType });
 }

package/dist/core/pricing.js

@@ -43,6 +43,8 @@ export function getModelPricing(model) {
  * Returns 0 if model pricing is not found.
  */
 export function calculateCost(model, inputTokens, outputTokens) {
+    if (inputTokens < 0 || outputTokens < 0)
+        return 0;
     const pricing = getModelPricing(model);
     if (!pricing)
         return 0;

package/dist/core/spans.js

@@ -1,12 +1,18 @@
 import { createHash } from "node:crypto";
-import { trace, SpanStatusCode } from "@opentelemetry/api";
+import { trace, diag, SpanStatusCode } from "@opentelemetry/api";
 import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "../types/index.js";
 import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordBudgetExceeded, recordBudgetBlocked, recordBudgetDowngraded, } from "./metrics.js";
 import { getConfig, getBudgetTracker } from "./tracer.js";
 import { GEN_AI_ATTRS as ATTRS } from "../types/index.js";
 const tracer = trace.getTracer(INSTRUMENTATION_NAME);
+let saltWarningEmitted = false;
 function sha256(text) {
-    const salt = getConfig()?.salt ?? "";
+    const config = getConfig();
+    const salt = config?.salt ?? "";
+    if (config?.hashContent && !config.salt && !saltWarningEmitted) {
+        diag.warn("toad-eye: hashContent is enabled without salt — short strings may be reversible. Set salt in config for stronger privacy.");
+        saltWarningEmitted = true;
+    }
     return createHash("sha256")
         .update(salt + text)
         .digest("hex");

package/dist/core/tracer.js

@@ -73,7 +73,10 @@ export function initObservability(config) {
     currentConfig = config;
     initMetrics();
     if (isCloudMode) {
-        console.log(`toad-eye: cloud mode enabled → sending telemetry to ${endpoint}`);
+        const masked = config.apiKey
+            ? config.apiKey.slice(0, 8) + "..." + config.apiKey.slice(-4)
+            : "";
+        console.log(`toad-eye: cloud mode enabled (${masked}) → sending telemetry to ${endpoint}`);
     }
     if (config.budgets) {
         budgetTracker = new BudgetTracker(config.budgets, config.onBudgetExceeded ?? "warn", config.downgradeCallback);

package/dist/drift/monitor.js

@@ -39,7 +39,11 @@ export function createDriftMonitor(config) {
     return {
         check: performCheck,
         checkInBackground(response, providerName, model) {
-            void performCheck(response, providerName, model).catch((err) => {
+            const timeout = new Promise((_, reject) => setTimeout(() => reject(new Error("Drift check timeout (5s)")), 5000));
+            void Promise.race([
+                performCheck(response, providerName, model),
+                timeout,
+            ]).catch((err) => {
                 console.warn(`[toad-eye] Drift check failed (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
             });
         },

package/dist/drift/openai-embeddings.js

@@ -20,7 +20,11 @@ export function createOpenAIEmbeddingProvider(apiKey, model = DEFAULT_MODEL) {
                 throw new Error(`OpenAI Embeddings API error: ${response.status} ${response.statusText}`);
             }
             const body = (await response.json());
-            return body.data[0].embedding;
+            const embedding = body.data?.[0]?.embedding;
+            if (!embedding) {
+                throw new Error("OpenAI Embeddings API returned no embeddings");
+            }
+            return embedding;
         },
     };
 }

package/dist/export.js

@@ -31,7 +31,7 @@ function buildAssertions(completion) {
             value: Math.ceil(completion.length * LENGTH_BUFFER_MULTIPLIER),
         });
         if (!completion.toLowerCase().includes(REFUSAL_MARKER)) {
-            assertions.push({ type: "not_contains", value: "I cannot" });
+            assertions.push({ type: "not_contains", value: REFUSAL_MARKER });
         }
         if (isJson(completion)) {
             assertions.push({ type: "is_json", value: true });

package/dist/guard.d.ts

@@ -2,10 +2,9 @@
  * Shadow Guardrails integration — records toad-guard validation results
  * as span attributes and metrics without blocking LLM responses.
  *
- * toad-guard validates each LLM response and passes the result here.
- * In shadow mode, validation runs but never throws — results are recorded
- * for observability only, letting you tune guardrail thresholds on
- * production traffic before switching to enforce mode.
+ * Results are recorded as attributes on the currently active span
+ * (typically the LLM call span from traceLLMCall). This avoids
+ * trace bloat — no extra child spans per guard rule.
  *
  * Usage from toad-guard:
  * ```ts
@@ -17,10 +16,9 @@
  */
 import type { GuardResult } from "./types/index.js";
 /**
- * Record a guard validation result as span attributes and metrics.
+ * Record a guard validation result as attributes on the active span.
  *
- * Attaches guard attributes to a child span and increments counters.
- * If the guard failed, also increments the `would_block` counter —
- * useful for measuring how often shadow guardrails would have blocked.
+ * Attaches guard attributes directly to the current span (no child span created).
+ * Increments guard evaluation counter, and would_block counter if failed.
  */
 export declare function recordGuardResult(result: GuardResult): void;

package/dist/guard.js

@@ -2,10 +2,9 @@
  * Shadow Guardrails integration — records toad-guard validation results
  * as span attributes and metrics without blocking LLM responses.
  *
- * toad-guard validates each LLM response and passes the result here.
- * In shadow mode, validation runs but never throws — results are recorded
- * for observability only, letting you tune guardrail thresholds on
- * production traffic before switching to enforce mode.
+ * Results are recorded as attributes on the currently active span
+ * (typically the LLM call span from traceLLMCall). This avoids
+ * trace bloat — no extra child spans per guard rule.
  *
  * Usage from toad-guard:
  * ```ts
@@ -16,28 +15,27 @@
  * ```
  */
 import { trace } from "@opentelemetry/api";
-import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "./types/index.js";
+import { GEN_AI_ATTRS } from "./types/index.js";
 import { recordGuardEvaluation, recordGuardWouldBlock, } from "./core/metrics.js";
-const tracer = trace.getTracer(INSTRUMENTATION_NAME);
 /**
- * Record a guard validation result as span attributes and metrics.
+ * Record a guard validation result as attributes on the active span.
  *
- * Attaches guard attributes to a child span and increments counters.
- * If the guard failed, also increments the `would_block` counter —
- * useful for measuring how often shadow guardrails would have blocked.
+ * Attaches guard attributes directly to the current span (no child span created).
+ * Increments guard evaluation counter, and would_block counter if failed.
  */
 export function recordGuardResult(result) {
-    const span = tracer.startSpan(`guard.evaluate.${result.ruleName}`);
-    span.setAttributes({
-        [GEN_AI_ATTRS.GUARD_MODE]: result.mode,
-        [GEN_AI_ATTRS.GUARD_PASSED]: result.passed,
-        [GEN_AI_ATTRS.GUARD_RULE_NAME]: result.ruleName,
-        ...(!result.passed &&
-            result.failureReason !== undefined && {
-            [GEN_AI_ATTRS.GUARD_FAILURE_REASON]: result.failureReason,
-        }),
-    });
-    span.end();
+    const activeSpan = trace.getActiveSpan();
+    if (activeSpan) {
+        activeSpan.setAttributes({
+            [GEN_AI_ATTRS.GUARD_MODE]: result.mode,
+            [GEN_AI_ATTRS.GUARD_PASSED]: result.passed,
+            [GEN_AI_ATTRS.GUARD_RULE_NAME]: result.ruleName,
+            ...(!result.passed &&
+                result.failureReason !== undefined && {
+                [GEN_AI_ATTRS.GUARD_FAILURE_REASON]: result.failureReason,
+            }),
+        });
+    }
     recordGuardEvaluation(result.ruleName);
     if (!result.passed) {
         recordGuardWouldBlock(result.ruleName);

package/dist/instrumentations/anthropic.js

@@ -46,24 +46,17 @@ const messagesCreate = {
         };
     },
     isStreaming: (body) => !!body?.stream,
-    extractStreamResponse: (chunks) => {
-        // Anthropic stream events: message_start, content_block_delta, message_delta, message_stop
-        let completion = "";
-        let inputTokens = 0;
-        let outputTokens = 0;
-        for (const chunk of chunks) {
-            const event = chunk;
-            if (event.type === "content_block_delta" && event.delta?.text) {
-                completion += event.delta.text;
-            }
-            if (event.type === "message_start" && event.message?.usage) {
-                inputTokens = event.message.usage.input_tokens ?? 0;
-            }
-            if (event.type === "message_delta" && event.usage) {
-                outputTokens = event.usage.output_tokens ?? 0;
-            }
+    accumulateChunk: (acc, chunk) => {
+        const event = chunk;
+        if (event.type === "content_block_delta" && event.delta?.text) {
+            acc.completion += event.delta.text;
+        }
+        if (event.type === "message_start" && event.message?.usage) {
+            acc.inputTokens = event.message.usage.input_tokens ?? 0;
+        }
+        if (event.type === "message_delta" && event.usage) {
+            acc.outputTokens = event.usage.output_tokens ?? 0;
         }
-        return { completion, inputTokens, outputTokens };
     },
 };
 register(createInstrumentation({

package/dist/instrumentations/create.d.ts

@@ -1,9 +1,5 @@
 import type { LLMProvider } from "../types/index.js";
 import type { Instrumentation, PatchTarget } from "./types.js";
-/**
- * Factory for creating SDK instrumentations.
- * Eliminates boilerplate: SDK resolution, prototype patching, error handling, cleanup.
- */
 export declare function createInstrumentation(config: {
     name: LLMProvider;
     moduleName: string;

package/dist/instrumentations/create.js

@@ -1,8 +1,8 @@
 import { createRequire } from "node:module";
-import { trace, diag, SpanStatusCode } from "@opentelemetry/api";
+import { trace, context, diag, SpanStatusCode, } from "@opentelemetry/api";
 import { traceLLMCall } from "../core/spans.js";
 import { calculateCost } from "../core/pricing.js";
-import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, } from "../core/metrics.js";
+import { recordRequestDuration, recordRequestCost, recordTokens, recordRequest, recordError, recordTimeToFirstToken, } from "../core/metrics.js";
 import { GEN_AI_ATTRS, INSTRUMENTATION_NAME } from "../types/index.js";
 const require = createRequire(import.meta.url);
 function isModuleInstalled(moduleName) {
@@ -21,24 +21,43 @@ function loadModule(moduleName) {
     return sdk.default ?? sdk;
 }
 /**
- * Wrap an async iterable stream to intercept chunks.
- * Yields every chunk transparently to the caller while collecting them.
- * On stream end, calls onComplete with all accumulated chunks.
+ * Wrap an async iterable stream with an accumulator.
+ * Yields every chunk transparently. Accumulates only extracted data (no raw chunks stored).
+ * On stream end, calls onComplete with the accumulated result.
+ * onFirstChunk is called once when the first chunk arrives (for TTFT tracking).
  */
-async function* wrapAsyncIterable(stream, onChunk, onComplete, onError) {
-    const chunks = [];
+async function* wrapAsyncIterable(stream, accumulate, onFirstChunk, onComplete, onError) {
+    const acc = {
+        completion: "",
+        inputTokens: 0,
+        outputTokens: 0,
+    };
+    let firstChunk = true;
+    let completed = false;
+    let errored = false;
     try {
         for await (const chunk of stream) {
-            chunks.push(chunk);
-            onChunk(chunk);
+            if (firstChunk) {
+                onFirstChunk();
+                firstChunk = false;
+            }
+            accumulate(acc, chunk);
             yield chunk;
         }
-        onComplete(chunks);
+        completed = true;
+        onComplete(acc);
     }
     catch (err) {
+        errored = true;
         onError(err);
         throw err;
     }
+    finally {
+        // If stream was abandoned (consumer broke out), still record what we have
+        if (!completed && !errored) {
+            onComplete(acc);
+        }
+    }
 }
 function createStreamingHandler(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -48,32 +67,45 @@ original, patch, providerName) {
         const req = patch.extractRequest(body);
         const start = performance.now();
         const response = await original.call(thisArg, body, ...rest);
-        // The response should be an async iterable (Stream, MessageStream, etc.)
-        // We wrap it to intercept chunks
         const span = tracer.startSpan(`gen_ai.${providerName}.${req.model}`);
+        const ctx = trace.setSpan(context.active(), span);
         span.setAttributes({
             [GEN_AI_ATTRS.PROVIDER]: providerName,
             [GEN_AI_ATTRS.REQUEST_MODEL]: req.model,
             [GEN_AI_ATTRS.TEMPERATURE]: req.temperature ?? 1.0,
             [GEN_AI_ATTRS.OPERATION]: "chat",
         });
-        const wrapped = wrapAsyncIterable(response, () => { }, (chunks) => {
+        // Some SDKs (Gemini) return { stream: AsyncIterable } instead of a direct AsyncIterable
+        const resp = response;
+        const hasStreamProp = resp != null &&
+            typeof resp === "object" &&
+            "stream" in resp &&
+            resp["stream"] != null &&
+            typeof resp["stream"] === "object" &&
+            Symbol.asyncIterator in resp["stream"];
+        const streamIterable = hasStreamProp
+            ? resp["stream"]
+            : response;
+        const wrapped = context.bind(ctx, wrapAsyncIterable(streamIterable, (acc, chunk) => patch.accumulateChunk(acc, chunk), () => {
+            const ttft = performance.now() - start;
+            recordTimeToFirstToken(ttft, providerName, req.model);
+        }, (acc) => {
             const duration = performance.now() - start;
-            const res = patch.extractStreamResponse(chunks);
-            const cost = calculateCost(req.model, res.inputTokens, res.outputTokens);
+            const cost = calculateCost(req.model, acc.inputTokens, acc.outputTokens);
             span.setAttributes({
                 [GEN_AI_ATTRS.RESPONSE_MODEL]: req.model,
-                [GEN_AI_ATTRS.INPUT_TOKENS]: res.inputTokens,
-                [GEN_AI_ATTRS.OUTPUT_TOKENS]: res.outputTokens,
+                [GEN_AI_ATTRS.INPUT_TOKENS]: acc.inputTokens,
+                [GEN_AI_ATTRS.OUTPUT_TOKENS]: acc.outputTokens,
                 [GEN_AI_ATTRS.COST]: cost,
                 [GEN_AI_ATTRS.STATUS]: "success",
+                [GEN_AI_ATTRS.FINISH_REASONS]: ["stop"],
             });
             span.setStatus({ code: SpanStatusCode.OK });
             span.end();
             recordRequest(providerName, req.model);
             recordRequestDuration(duration, providerName, req.model);
             recordRequestCost(cost, providerName, req.model);
-            recordTokens(res.inputTokens + res.outputTokens, providerName, req.model);
+            recordTokens(acc.inputTokens + acc.outputTokens, providerName, req.model);
         }, (err) => {
             const duration = performance.now() - start;
             const message = err instanceof Error ? err.message : String(err);
@@ -86,26 +118,29 @@ original, patch, providerName) {
             recordRequest(providerName, req.model);
             recordRequestDuration(duration, providerName, req.model);
             recordError(providerName, req.model);
+        }));
+        // Proxy preserves the original object shape (getters, private fields, methods)
+        // while intercepting only the async iterator for instrumentation
+        return new Proxy(response, {
+            get(target, prop, receiver) {
+                if (prop === Symbol.asyncIterator) {
+                    return () => wrapped[Symbol.asyncIterator]();
+                }
+                return Reflect.get(target, prop, receiver);
+            },
         });
-        // Return an object that looks like the original stream but with our wrapper
-        // Preserve the original object shape — SDKs may have extra methods/properties
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const proxy = Object.create(response);
-        proxy[Symbol.asyncIterator] = () => wrapped[Symbol.asyncIterator]();
-        return proxy;
     };
 }
 function createPatchedMethod(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 original, patch, providerName) {
-    const streamHandler = patch.isStreaming && patch.extractStreamResponse
+    const streamHandler = patch.isStreaming && patch.accumulateChunk
         ? createStreamingHandler(original, patch, providerName)
         : null;
     return function patchedMethod(body, ...rest) {
         if (patch.shouldSkip?.(body)) {
             return original.call(this, body, ...rest);
         }
-        // Handle streaming calls
         if (streamHandler && patch.isStreaming?.(body)) {
             return streamHandler(this, body, rest);
         }
@@ -127,10 +162,6 @@ original, patch, providerName) {
         });
     };
 }
-/**
- * Factory for creating SDK instrumentations.
- * Eliminates boilerplate: SDK resolution, prototype patching, error handling, cleanup.
- */
 export function createInstrumentation(config) {
     const activePatches = [];
     return {

package/dist/instrumentations/gemini.js

@@ -21,7 +21,6 @@ const generateContent = {
     extractRequest(body) {
         return {
             prompt: extractPrompt(body),
-            // model comes from `this.model` — injected by create.ts via the patched function's `this`
             model: "unknown",
         };
     },
@@ -41,8 +40,42 @@ const generateContent = {
         };
     },
 };
+const generateContentStream = {
+    getPrototype: (sdk) => sdk?.GenerativeModel?.prototype,
+    method: "generateContentStream",
+    extractRequest(body) {
+        return {
+            prompt: extractPrompt(body),
+            model: "unknown",
+        };
+    },
+    extractResponse: () => ({
+        completion: "",
+        inputTokens: 0,
+        outputTokens: 0,
+    }),
+    // generateContentStream always returns a stream
+    isStreaming: () => true,
+    accumulateChunk: (acc, chunk) => {
+        // Gemini stream chunks are GenerateContentResponse objects
+        const c = chunk;
+        try {
+            const text = c?.text?.();
+            if (text)
+                acc.completion += text;
+        }
+        catch {
+            // text() may throw if content is blocked
+        }
+        if (c?.usageMetadata) {
+            acc.inputTokens = c.usageMetadata.promptTokenCount ?? acc.inputTokens;
+            acc.outputTokens =
+                c.usageMetadata.candidatesTokenCount ?? acc.outputTokens;
+        }
+    },
+};
 register(createInstrumentation({
     name: "gemini",
     moduleName: "@google/generative-ai",
-    patches: [generateContent],
+    patches: [generateContent, generateContentStream],
 }));

package/dist/instrumentations/openai.js

@@ -46,23 +46,15 @@ const chatCompletions = {
         };
     },
     isStreaming: (body) => !!body?.stream,
-    extractStreamResponse: (chunks) => {
-        // OpenAI stream chunks: { choices: [{ delta: { content } }], usage?: { ... } }
-        let completion = "";
-        let inputTokens = 0;
-        let outputTokens = 0;
-        for (const chunk of chunks) {
-            const c = chunk;
-            const delta = c?.choices?.[0]?.delta?.content;
-            if (delta)
-                completion += delta;
-            // Usage is typically in the final chunk (when stream_options.include_usage is set)
-            if (c?.usage) {
-                inputTokens = c.usage.prompt_tokens ?? inputTokens;
-                outputTokens = c.usage.completion_tokens ?? outputTokens;
-            }
+    accumulateChunk: (acc, chunk) => {
+        const c = chunk;
+        const delta = c?.choices?.[0]?.delta?.content;
+        if (delta)
+            acc.completion += delta;
+        if (c?.usage) {
+            acc.inputTokens = c.usage.prompt_tokens ?? acc.inputTokens;
+            acc.outputTokens = c.usage.completion_tokens ?? acc.outputTokens;
         }
-        return { completion, inputTokens, outputTokens };
     },
 };
 const embeddings = {

package/dist/instrumentations/types.d.ts

@@ -6,6 +6,12 @@ export interface Instrumentation {
     /** Restore original methods. */
     disable(): void;
 }
+/** Accumulated stream data — only primitives, no raw SDK objects. */
+export interface StreamAccumulator {
+    completion: string;
+    inputTokens: number;
+    outputTokens: number;
+}
 /** Describes a single method to monkey-patch on an SDK prototype. */
 export interface PatchTarget {
     /** How to find the prototype from the loaded SDK module */
@@ -28,10 +34,6 @@ export interface PatchTarget {
     shouldSkip?: (body: unknown) => boolean;
     /** Return true if this call is a streaming request */
     isStreaming?: (body: unknown) => boolean;
-    /** Extract final stats from accumulated stream chunks. Required when isStreaming is set. */
-    extractStreamResponse?: (chunks: unknown[]) => {
-        completion: string;
-        inputTokens: number;
-        outputTokens: number;
-    };
+    /** Process one stream chunk — extract only the data we need into the accumulator. */
+    accumulateChunk?: (acc: StreamAccumulator, chunk: unknown) => void;
 }

package/dist/types/metrics.d.ts

@@ -17,6 +17,7 @@ export declare const GEN_AI_METRICS: {
     readonly GUARD_EVALUATIONS: "gen_ai.toad_eye.guard.evaluations";
     readonly GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block";
     readonly SEMANTIC_DRIFT: "gen_ai.toad_eye.semantic_drift";
+    readonly TIME_TO_FIRST_TOKEN: "gen_ai.client.time_to_first_token";
     readonly BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded";
     readonly BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked";
     readonly BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded";

package/dist/types/metrics.js

@@ -20,6 +20,8 @@ export const GEN_AI_METRICS = {
     GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block",
     // Drift metrics
     SEMANTIC_DRIFT: "gen_ai.toad_eye.semantic_drift",
+    // Streaming metrics
+    TIME_TO_FIRST_TOKEN: "gen_ai.client.time_to_first_token",
     // Budget metrics
     BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded",
     BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toad-eye",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
   "type": "module",
   "main": "dist/index.js",

package/templates/grafana/dashboards/provider-health.json

@@ -0,0 +1,191 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Provider Status",
+      "description": "healthy (< 10% errors) / degraded (10-50%) / down (> 50%)",
+      "type": "stat",
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 0.5 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "range",
+              "options": {
+                "from": 0,
+                "to": 0.1,
+                "result": { "text": "HEALTHY", "color": "green" }
+              }
+            },
+            {
+              "type": "range",
+              "options": {
+                "from": 0.1,
+                "to": 0.5,
+                "result": { "text": "DEGRADED", "color": "yellow" }
+              }
+            },
+            {
+              "type": "range",
+              "options": {
+                "from": 0.5,
+                "to": 1,
+                "result": { "text": "DOWN", "color": "red" }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Request Volume by Provider",
+      "type": "timeseries",
+      "gridPos": { "h": 6, "w": 16, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 15 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Error Rate by Provider (5m window)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}} error rate",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "custom": { "fillOpacity": 10 },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 0.5 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Error Breakdown by Type",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name, error_type) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}} — {{error_type}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Provider Availability (24h / 7d / 30d)",
+      "description": "Percentage of time with error rate below 10%",
+      "type": "table",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "1 - (sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total[24h])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total[24h])))",
+          "format": "table",
+          "instant": true,
+          "refId": "uptime_24h"
+        },
+        {
+          "expr": "1 - (sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total[7d])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total[7d])))",
+          "format": "table",
+          "instant": true,
+          "refId": "uptime_7d"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "renameByName": {
+              "gen_ai_provider_name": "Provider",
+              "Value #uptime_24h": "Uptime 24h",
+              "Value #uptime_7d": "Uptime 7d"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "percentunit", "decimals": 2 },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["toad-eye", "provider-health"],
+  "templating": {
+    "list": [
+      {
+        "name": "provider",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" },
+        "regex": ""
+      }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "utc",
+  "title": "toad-eye / Provider Health",
+  "uid": "toad-eye-provider-health",
+  "version": 1
+}