toad-eye 2.2.0 → 2.3.0

package/dist/instrumentations/registry.js

@@ -1,25 +1,48 @@
-import { diag } from "@opentelemetry/api";
 const instrumentations = new Map();
 const active = new Set();
 export function register(inst) {
     instrumentations.set(inst.name, inst);
 }
-export function enableAll(providers) {
+/**
+ * Lazy registration — loads provider modules and registers instrumentations
+ * only when enableAll() is first called. This avoids pulling in create.ts,
+ * metrics.ts, and node:module on every `import` of toad-eye.
+ */
+let registered = false;
+/** Skip lazy registration (used in tests that register mocks manually). */
+export function skipAutoRegister() {
+    registered = true;
+}
+async function ensureRegistered() {
+    if (registered)
+        return;
+    registered = true;
+    const [openai, anthropic, gemini] = await Promise.all([
+        import("./openai.js"),
+        import("./anthropic.js"),
+        import("./gemini.js"),
+    ]);
+    register(openai.openaiInstrumentation);
+    register(anthropic.anthropicInstrumentation);
+    register(gemini.geminiInstrumentation);
+}
+export async function enableAll(providers) {
+    await ensureRegistered();
+    const validProviders = Array.from(instrumentations.keys()).join(", ");
     for (const name of providers) {
         if (active.has(name))
             continue;
         const inst = instrumentations.get(name);
         if (!inst) {
-            diag.warn(`toad-eye: unknown provider "${name}", skipping`);
+            console.warn(`toad-eye: unknown provider "${name}" — valid providers: ${validProviders}`);
             continue;
         }
         const patched = inst.enable();
         if (patched) {
             active.add(name);
-            diag.debug(`toad-eye: auto-instrumented ${name}`);
         }
         else {
-            diag.debug(`toad-eye: ${name} SDK not found, skipping`);
+            console.warn(`toad-eye: "${name}" SDK not found — install it to enable auto-instrumentation: npm install ${name === "gemini" ? "@google/generative-ai" : name}`);
         }
     }
 }

package/dist/instrumentations/types.d.ts

@@ -18,8 +18,10 @@ export interface PatchTarget {
     getPrototype: (sdk: any) => any | undefined;
     /** Method name on the prototype to patch */
     method: string;
-    /** Extract LLMCallInput fields from the request arguments */
-    extractRequest: (body: unknown) => {
+    /** Extract LLMCallInput fields from the request arguments.
+     *  thisArg is the SDK object instance — use it when the model name is not in the request body
+     *  (e.g., Gemini's GenerativeModel stores it as this.model). */
+    extractRequest: (body: unknown, thisArg?: unknown) => {
         prompt: string;
         model: string;
         temperature?: number;

package/dist/types/attributes.d.ts

@@ -19,6 +19,9 @@ export declare const GEN_AI_ATTRS: {
     readonly AGENT_STEP_NUMBER: "gen_ai.agent.step.number";
     readonly AGENT_TOOL_NAME: "gen_ai.agent.tool.name";
     readonly AGENT_STEP_CONTENT: "gen_ai.agent.step.content";
+    readonly AGENT_HANDOFF_TO: "gen_ai.agent.handoff.to";
+    readonly AGENT_HANDOFF_REASON: "gen_ai.agent.handoff.reason";
+    readonly AGENT_LOOP_COUNT: "gen_ai.agent.loop_count";
     readonly GUARD_MODE: "gen_ai.toad_eye.guard.mode";
     readonly GUARD_PASSED: "gen_ai.toad_eye.guard.passed";
     readonly GUARD_FAILURE_REASON: "gen_ai.toad_eye.guard.failure_reason";

package/dist/types/attributes.js

@@ -21,6 +21,9 @@ export const GEN_AI_ATTRS = {
     AGENT_STEP_NUMBER: "gen_ai.agent.step.number",
     AGENT_TOOL_NAME: "gen_ai.agent.tool.name",
     AGENT_STEP_CONTENT: "gen_ai.agent.step.content",
+    AGENT_HANDOFF_TO: "gen_ai.agent.handoff.to",
+    AGENT_HANDOFF_REASON: "gen_ai.agent.handoff.reason",
+    AGENT_LOOP_COUNT: "gen_ai.agent.loop_count",
     // Guard (shadow mode) attributes
     GUARD_MODE: "gen_ai.toad_eye.guard.mode",
     GUARD_PASSED: "gen_ai.toad_eye.guard.passed",

package/dist/types/config.d.ts

@@ -1,5 +1,19 @@
-import type { LLMProvider } from "./providers.js";
+import type { InstrumentTarget } from "./providers.js";
 import type { BudgetConfig, BudgetExceededMode, DowngradeCallback } from "../budget/types.js";
+/** Sampling configuration. Tail sampling runs in OTel Collector, SDK sends all spans. */
+export interface SamplingConfig {
+    /** SDK-side sampling rate (0.0–1.0). Default: 1.0 (send everything to Collector). */
+    readonly sdkRate?: number | undefined;
+    /** Collector-side tail sampling settings (used when generating otel-collector.yml). */
+    readonly collector?: {
+        /** Keep 100% of error traces. Default: true */
+        readonly keepErrors?: boolean | undefined;
+        /** Keep 100% of traces slower than this (ms). Default: 2000 */
+        readonly highLatencyMs?: number | undefined;
+        /** Sample this % of healthy traffic (0–100). Default: 10 */
+        readonly healthyRate?: number | undefined;
+    } | undefined;
+}
 /**
  * Configuration for initObservability().
  * This is what the user passes when connecting toad-eye to their service.
@@ -20,7 +34,13 @@ export interface ToadEyeConfig {
     readonly salt?: string | undefined;
     /** Regex patterns to redact from prompt/completion text before recording. */
     readonly redactPatterns?: readonly RegExp[] | undefined;
-    readonly instrument?: readonly LLMProvider[] | undefined;
+    /** Enable built-in redaction patterns for common PII: email, SSN, credit card, phone. */
+    readonly redactDefaults?: boolean | undefined;
+    /** Log what was masked to console (for debugging redaction config). No data is sent externally. */
+    readonly auditMasking?: boolean | undefined;
+    /** Configure trace sampling. Tail sampling happens in OTel Collector, not SDK. */
+    readonly sampling?: SamplingConfig | undefined;
+    readonly instrument?: readonly InstrumentTarget[] | undefined;
     /** Budget limits — daily, per-user, per-model spend caps in USD. */
     readonly budgets?: BudgetConfig | undefined;
     /** What to do when budget is exceeded: 'warn' (log), 'block' (throw), 'downgrade' (callback). */

package/dist/types/index.d.ts

@@ -9,9 +9,9 @@
  * - spans.ts      — span-related types (LLMSpanAttributes, SpanStatus)
  */
 export declare const INSTRUMENTATION_NAME = "toad-eye";
-export type { LLMProvider } from "./providers.js";
-export type { ToadEyeConfig } from "./config.js";
+export type { LLMProvider, InstrumentTarget } from "./providers.js";
+export type { ToadEyeConfig, SamplingConfig } from "./config.js";
 export { GEN_AI_ATTRS, LLM_ATTRS } from "./attributes.js";
 export { GEN_AI_METRICS, LLM_METRICS } from "./metrics.js";
 export type { MetricName } from "./metrics.js";
-export type { SpanStatus, LLMSpanAttributes, AgentStepType, AgentStepInput, GuardMode, GuardResult, } from "./spans.js";
+export type { SpanStatus, LLMSpanAttributes, AgentStepType, AgentStepInput, AgentQueryOptions, GuardMode, GuardResult, } from "./spans.js";

package/dist/types/metrics.d.ts

@@ -21,6 +21,8 @@ export declare const GEN_AI_METRICS: {
     readonly BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded";
     readonly BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked";
     readonly BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded";
+    readonly RESPONSE_EMPTY: "gen_ai.toad_eye.response.empty";
+    readonly RESPONSE_LATENCY_PER_TOKEN: "gen_ai.toad_eye.response.latency_per_token";
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export declare const LLM_METRICS: {

package/dist/types/metrics.js

@@ -26,6 +26,9 @@ export const GEN_AI_METRICS = {
     BUDGET_EXCEEDED: "gen_ai.toad_eye.budget.exceeded",
     BUDGET_BLOCKED: "gen_ai.toad_eye.budget.blocked",
     BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded",
+    // Response quality proxy metrics
+    RESPONSE_EMPTY: "gen_ai.toad_eye.response.empty",
+    RESPONSE_LATENCY_PER_TOKEN: "gen_ai.toad_eye.response.latency_per_token",
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export const LLM_METRICS = {

package/dist/types/providers.d.ts

@@ -2,4 +2,9 @@
  * LLM providers supported by toad-eye.
  * Values follow OTel GenAI semconv `gen_ai.provider.name`.
  */
-export type LLMProvider = "anthropic" | "gemini" | "openai";
+export type LLMProvider = "anthropic" | "gemini" | "openai" | (string & {});
+/**
+ * All instrumentable targets — includes LLM providers + framework SDKs.
+ * 'ai' = Vercel AI SDK (uses SpanProcessor, not monkey-patching).
+ */
+export type InstrumentTarget = LLMProvider | "ai";

package/dist/types/spans.d.ts

@@ -1,14 +1,23 @@
 import type { LLMProvider } from "./providers.js";
 /** Span status values used in trace attributes. */
 export type SpanStatus = "success" | "error";
-/** ReAct agent step types: think → act → observe → answer */
-export type AgentStepType = "think" | "act" | "observe" | "answer";
+/** ReAct agent step types: think → act → observe → answer → handoff */
+export type AgentStepType = "think" | "act" | "observe" | "answer" | "handoff";
 /** Input for traceAgentStep — describes a single agent step */
 export interface AgentStepInput {
     readonly type: AgentStepType;
     readonly stepNumber: number;
     readonly content?: string | undefined;
     readonly toolName?: string | undefined;
+    /** Target agent name for handoff steps */
+    readonly toAgent?: string | undefined;
+    /** Reason for handoff */
+    readonly handoffReason?: string | undefined;
+}
+/** Options for traceAgentQuery */
+export interface AgentQueryOptions {
+    /** Max steps before recording a warning. Default: 25 */
+    readonly maxSteps?: number | undefined;
 }
 /** Guard execution mode */
 export type GuardMode = "shadow" | "enforce";

package/dist/vercel.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Vercel AI SDK integration — enriches AI SDK OTel spans with toad-eye
+ * cost tracking and metrics. No monkey-patching needed.
+ *
+ * The Vercel AI SDK already emits OTel spans when `experimental_telemetry`
+ * is enabled. This module adds a SpanProcessor that intercepts those spans
+ * and calculates cost from token usage.
+ *
+ * Usage:
+ * ```ts
+ * import { initObservability } from 'toad-eye';
+ * import { withToadEye } from 'toad-eye/vercel';
+ *
+ * initObservability({ serviceName: 'my-app' });
+ *
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye(),
+ * });
+ * ```
+ */
+import type { SpanProcessor, ReadableSpan } from "@opentelemetry/sdk-trace-base";
+/**
+ * OTel SpanProcessor that enriches Vercel AI SDK spans with toad-eye
+ * cost attributes and records metrics.
+ */
+export declare class ToadEyeAISpanProcessor implements SpanProcessor {
+    forceFlush(): Promise<void>;
+    shutdown(): Promise<void>;
+    onStart(): void;
+    onEnd(span: ReadableSpan): void;
+}
+/**
+ * Helper for `experimental_telemetry` option — enables telemetry with
+ * optional toad-eye metadata.
+ *
+ * ```ts
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye({ functionId: 'my-feature' }),
+ * });
+ * ```
+ */
+export declare function withToadEye(options?: {
+    functionId?: string;
+    metadata?: Record<string, string>;
+}): {
+    functionId?: string;
+    metadata?: Record<string, string>;
+    isEnabled: true;
+};

package/dist/vercel.js

@@ -0,0 +1,106 @@
+/**
+ * Vercel AI SDK integration — enriches AI SDK OTel spans with toad-eye
+ * cost tracking and metrics. No monkey-patching needed.
+ *
+ * The Vercel AI SDK already emits OTel spans when `experimental_telemetry`
+ * is enabled. This module adds a SpanProcessor that intercepts those spans
+ * and calculates cost from token usage.
+ *
+ * Usage:
+ * ```ts
+ * import { initObservability } from 'toad-eye';
+ * import { withToadEye } from 'toad-eye/vercel';
+ *
+ * initObservability({ serviceName: 'my-app' });
+ *
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye(),
+ * });
+ * ```
+ */
+import { calculateCost } from "./core/pricing.js";
+import { recordRequestCost, recordTokens, recordRequest, recordRequestDuration, } from "./core/metrics.js";
+// Vercel AI SDK span operation names
+const AI_SDK_OPERATIONS = new Set([
+    "ai.generateText",
+    "ai.streamText",
+    "ai.generateObject",
+    "ai.streamObject",
+]);
+function isAiSdkSpan(span) {
+    const opId = span.attributes["ai.operationId"];
+    if (typeof opId === "string")
+        return AI_SDK_OPERATIONS.has(opId);
+    return AI_SDK_OPERATIONS.has(span.name);
+}
+function getStringAttr(span, key) {
+    const val = span.attributes[key];
+    return typeof val === "string" ? val : undefined;
+}
+function getNumberAttr(span, key) {
+    const val = span.attributes[key];
+    return typeof val === "number" ? val : 0;
+}
+/**
+ * OTel SpanProcessor that enriches Vercel AI SDK spans with toad-eye
+ * cost attributes and records metrics.
+ */
+export class ToadEyeAISpanProcessor {
+    forceFlush() {
+        return Promise.resolve();
+    }
+    shutdown() {
+        return Promise.resolve();
+    }
+    onStart() {
+        // no-op — we process on end
+    }
+    onEnd(span) {
+        if (!isAiSdkSpan(span))
+            return;
+        const model = getStringAttr(span, "gen_ai.request.model") ??
+            getStringAttr(span, "ai.model.id") ??
+            "unknown";
+        const provider = getStringAttr(span, "gen_ai.system") ??
+            getStringAttr(span, "ai.model.provider")?.split(".")[0] ??
+            "unknown";
+        const inputTokens = getNumberAttr(span, "gen_ai.usage.input_tokens") ||
+            getNumberAttr(span, "ai.usage.promptTokens");
+        const outputTokens = getNumberAttr(span, "gen_ai.usage.output_tokens") ||
+            getNumberAttr(span, "ai.usage.completionTokens");
+        // Calculate cost from token usage
+        const cost = calculateCost(model, inputTokens, outputTokens);
+        // We can't modify a ReadableSpan's attributes directly, but we can
+        // record metrics which is the primary value-add
+        const durationMs = (span.endTime[0] - span.startTime[0]) * 1000 +
+            (span.endTime[1] - span.startTime[1]) / 1_000_000;
+        recordRequest(provider, model);
+        recordRequestDuration(durationMs, provider, model);
+        if (cost > 0) {
+            recordRequestCost(cost, provider, model);
+        }
+        if (inputTokens + outputTokens > 0) {
+            recordTokens(inputTokens + outputTokens, provider, model);
+        }
+    }
+}
+/**
+ * Helper for `experimental_telemetry` option — enables telemetry with
+ * optional toad-eye metadata.
+ *
+ * ```ts
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye({ functionId: 'my-feature' }),
+ * });
+ * ```
+ */
+export function withToadEye(options) {
+    return {
+        isEnabled: true,
+        ...options,
+    };
+}

package/package.json

@@ -1,10 +1,32 @@
 {
   "name": "toad-eye",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./alerts": {
+      "import": "./dist/alerts/index.js",
+      "types": "./dist/alerts/index.d.ts"
+    },
+    "./drift": {
+      "import": "./dist/drift/index.js",
+      "types": "./dist/drift/index.d.ts"
+    },
+    "./export": {
+      "import": "./dist/export.js",
+      "types": "./dist/export.d.ts"
+    },
+    "./vercel": {
+      "import": "./dist/vercel.js",
+      "types": "./dist/vercel.d.ts"
+    }
+  },
   "bin": {
     "toad-eye": "dist/cli.js"
   },
@@ -51,13 +73,15 @@
     "@opentelemetry/resources": "^2.6.0",
     "@opentelemetry/sdk-metrics": "^2.6.0",
     "@opentelemetry/sdk-node": "^0.213.0",
+    "@opentelemetry/sdk-trace-base": "^2.6.0",
+    "@opentelemetry/sdk-trace-node": "^2.6.0",
     "@opentelemetry/semantic-conventions": "^1.40.0",
-    "nodemailer": "^8.0.3",
     "yaml": "^2.7.0"
   },
   "peerDependencies": {
     "@anthropic-ai/sdk": ">=0.30.0",
     "@google/generative-ai": ">=0.20.0",
+    "nodemailer": ">=8.0.0",
     "openai": ">=4.0.0"
   },
   "peerDependenciesMeta": {
@@ -69,11 +93,13 @@
     },
     "@google/generative-ai": {
       "optional": true
+    },
+    "nodemailer": {
+      "optional": true
     }
   },
   "devDependencies": {
     "@types/nodemailer": "^7.0.11",
-    "playwright": "^1.58.2",
     "vitest": "^4.1.0"
   }
 }

package/templates/grafana/dashboards/agent-workflow.json

@@ -0,0 +1,207 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Avg Steps per Query",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_steps_per_query_sum) / sum(gen_ai_agent_steps_per_query_count) or vector(0)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 1,
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 10 },
+              { "color": "red", "value": 20 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Agent Queries",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_steps_per_query_count)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Tool Invocations",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_tool_usage_total)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Unique Tools Used",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "count(count by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agent Step Type Breakdown",
+      "description": "Distribution of think / act / observe / answer steps",
+      "type": "piechart",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_agent_step_type) (gen_ai_agent_steps_per_query_sum)",
+          "legendFormat": "{{gen_ai_agent_step_type}}",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "values": ["value", "percent"]
+        },
+        "pieType": "donut"
+      }
+    },
+    {
+      "title": "Tool Usage Frequency",
+      "description": "Invocations per tool name",
+      "type": "barchart",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sort_desc(sum by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
+          "legendFormat": "{{gen_ai_agent_tool_name}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "title": "Steps per Query Over Time",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(gen_ai_agent_steps_per_query_sum[5m])) / sum(rate(gen_ai_agent_steps_per_query_count[5m]))",
+          "legendFormat": "avg steps/query",
+          "refId": "avg"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_agent_steps_per_query_bucket[5m])))",
+          "legendFormat": "p95 steps/query",
+          "refId": "p95"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": { "fillOpacity": 10 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Tool Invocations Over Time",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_agent_tool_name) (rate(gen_ai_agent_tool_usage_total[5m]))",
+          "legendFormat": "{{gen_ai_agent_tool_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Steps Distribution Histogram",
+      "description": "How many steps do agent queries typically take?",
+      "type": "histogram",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (le) (gen_ai_agent_steps_per_query_bucket)",
+          "format": "heatmap",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "fillOpacity": 80 }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["toad-eye", "agent"],
+  "templating": {
+    "list": [
+      {
+        "name": "tool",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": "label_values(gen_ai_agent_tool_usage_total, gen_ai_agent_tool_name)",
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" },
+        "regex": ""
+      }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "utc",
+  "title": "toad-eye / Agent Workflow",
+  "uid": "toad-eye-agent-workflow",
+  "version": 1
+}

package/templates/otel-collector.yml

@@ -8,6 +8,30 @@ processors:
   batch:
     timeout: 1s
 
+  # Tail-based sampling: keeps 100% errors + slow traces, samples healthy traffic.
+  # Only applies to traces pipeline — metrics always get 100%.
+  # To disable: remove tail_sampling from traces pipeline processors.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 100000
+    expected_new_traces_per_sec: 100
+    policies:
+      # Keep all traces with errors
+      - name: errors-always
+        type: status_code
+        status_code:
+          status_codes: [ERROR]
+      # Keep all slow traces (> 2s)
+      - name: slow-traces
+        type: latency
+        latency:
+          threshold_ms: 2000
+      # Sample 10% of remaining healthy traffic
+      - name: probabilistic-fallback
+        type: probabilistic
+        probabilistic:
+          sampling_percentage: 10
+
 exporters:
   otlphttp/jaeger:
     endpoint: http://jaeger:4318
@@ -21,7 +45,7 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
+      processors: [tail_sampling, batch]
       exporters: [otlphttp/jaeger]
     metrics:
       receivers: [otlp]