toad-eye 2.1.1 → 2.3.0

package/dist/vercel.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Vercel AI SDK integration — enriches AI SDK OTel spans with toad-eye
+ * cost tracking and metrics. No monkey-patching needed.
+ *
+ * The Vercel AI SDK already emits OTel spans when `experimental_telemetry`
+ * is enabled. This module adds a SpanProcessor that intercepts those spans
+ * and calculates cost from token usage.
+ *
+ * Usage:
+ * ```ts
+ * import { initObservability } from 'toad-eye';
+ * import { withToadEye } from 'toad-eye/vercel';
+ *
+ * initObservability({ serviceName: 'my-app' });
+ *
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye(),
+ * });
+ * ```
+ */
+import type { SpanProcessor, ReadableSpan } from "@opentelemetry/sdk-trace-base";
+/**
+ * OTel SpanProcessor that enriches Vercel AI SDK spans with toad-eye
+ * cost attributes and records metrics.
+ */
+export declare class ToadEyeAISpanProcessor implements SpanProcessor {
+    forceFlush(): Promise<void>;
+    shutdown(): Promise<void>;
+    onStart(): void;
+    onEnd(span: ReadableSpan): void;
+}
+/**
+ * Helper for `experimental_telemetry` option — enables telemetry with
+ * optional toad-eye metadata.
+ *
+ * ```ts
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye({ functionId: 'my-feature' }),
+ * });
+ * ```
+ */
+export declare function withToadEye(options?: {
+    functionId?: string;
+    metadata?: Record<string, string>;
+}): {
+    functionId?: string;
+    metadata?: Record<string, string>;
+    isEnabled: true;
+};

package/dist/vercel.js

@@ -0,0 +1,106 @@
+/**
+ * Vercel AI SDK integration — enriches AI SDK OTel spans with toad-eye
+ * cost tracking and metrics. No monkey-patching needed.
+ *
+ * The Vercel AI SDK already emits OTel spans when `experimental_telemetry`
+ * is enabled. This module adds a SpanProcessor that intercepts those spans
+ * and calculates cost from token usage.
+ *
+ * Usage:
+ * ```ts
+ * import { initObservability } from 'toad-eye';
+ * import { withToadEye } from 'toad-eye/vercel';
+ *
+ * initObservability({ serviceName: 'my-app' });
+ *
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye(),
+ * });
+ * ```
+ */
+import { calculateCost } from "./core/pricing.js";
+import { recordRequestCost, recordTokens, recordRequest, recordRequestDuration, } from "./core/metrics.js";
+// Vercel AI SDK span operation names
+const AI_SDK_OPERATIONS = new Set([
+    "ai.generateText",
+    "ai.streamText",
+    "ai.generateObject",
+    "ai.streamObject",
+]);
+function isAiSdkSpan(span) {
+    const opId = span.attributes["ai.operationId"];
+    if (typeof opId === "string")
+        return AI_SDK_OPERATIONS.has(opId);
+    return AI_SDK_OPERATIONS.has(span.name);
+}
+function getStringAttr(span, key) {
+    const val = span.attributes[key];
+    return typeof val === "string" ? val : undefined;
+}
+function getNumberAttr(span, key) {
+    const val = span.attributes[key];
+    return typeof val === "number" ? val : 0;
+}
+/**
+ * OTel SpanProcessor that enriches Vercel AI SDK spans with toad-eye
+ * cost attributes and records metrics.
+ */
+export class ToadEyeAISpanProcessor {
+    forceFlush() {
+        return Promise.resolve();
+    }
+    shutdown() {
+        return Promise.resolve();
+    }
+    onStart() {
+        // no-op — we process on end
+    }
+    onEnd(span) {
+        if (!isAiSdkSpan(span))
+            return;
+        const model = getStringAttr(span, "gen_ai.request.model") ??
+            getStringAttr(span, "ai.model.id") ??
+            "unknown";
+        const provider = getStringAttr(span, "gen_ai.system") ??
+            getStringAttr(span, "ai.model.provider")?.split(".")[0] ??
+            "unknown";
+        const inputTokens = getNumberAttr(span, "gen_ai.usage.input_tokens") ||
+            getNumberAttr(span, "ai.usage.promptTokens");
+        const outputTokens = getNumberAttr(span, "gen_ai.usage.output_tokens") ||
+            getNumberAttr(span, "ai.usage.completionTokens");
+        // Calculate cost from token usage
+        const cost = calculateCost(model, inputTokens, outputTokens);
+        // We can't modify a ReadableSpan's attributes directly, but we can
+        // record metrics which is the primary value-add
+        const durationMs = (span.endTime[0] - span.startTime[0]) * 1000 +
+            (span.endTime[1] - span.startTime[1]) / 1_000_000;
+        recordRequest(provider, model);
+        recordRequestDuration(durationMs, provider, model);
+        if (cost > 0) {
+            recordRequestCost(cost, provider, model);
+        }
+        if (inputTokens + outputTokens > 0) {
+            recordTokens(inputTokens + outputTokens, provider, model);
+        }
+    }
+}
+/**
+ * Helper for `experimental_telemetry` option — enables telemetry with
+ * optional toad-eye metadata.
+ *
+ * ```ts
+ * const result = await generateText({
+ *   model: openai('gpt-4o'),
+ *   prompt: 'Hello',
+ *   experimental_telemetry: withToadEye({ functionId: 'my-feature' }),
+ * });
+ * ```
+ */
+export function withToadEye(options) {
+    return {
+        isEnabled: true,
+        ...options,
+    };
+}

package/package.json

@@ -1,10 +1,32 @@
 {
   "name": "toad-eye",
-  "version": "2.1.1",
+  "version": "2.3.0",
   "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./alerts": {
+      "import": "./dist/alerts/index.js",
+      "types": "./dist/alerts/index.d.ts"
+    },
+    "./drift": {
+      "import": "./dist/drift/index.js",
+      "types": "./dist/drift/index.d.ts"
+    },
+    "./export": {
+      "import": "./dist/export.js",
+      "types": "./dist/export.d.ts"
+    },
+    "./vercel": {
+      "import": "./dist/vercel.js",
+      "types": "./dist/vercel.d.ts"
+    }
+  },
   "bin": {
     "toad-eye": "dist/cli.js"
   },
@@ -51,13 +73,15 @@
     "@opentelemetry/resources": "^2.6.0",
     "@opentelemetry/sdk-metrics": "^2.6.0",
     "@opentelemetry/sdk-node": "^0.213.0",
+    "@opentelemetry/sdk-trace-base": "^2.6.0",
+    "@opentelemetry/sdk-trace-node": "^2.6.0",
     "@opentelemetry/semantic-conventions": "^1.40.0",
-    "nodemailer": "^8.0.3",
     "yaml": "^2.7.0"
   },
   "peerDependencies": {
     "@anthropic-ai/sdk": ">=0.30.0",
     "@google/generative-ai": ">=0.20.0",
+    "nodemailer": ">=8.0.0",
     "openai": ">=4.0.0"
   },
   "peerDependenciesMeta": {
@@ -69,11 +93,13 @@
     },
     "@google/generative-ai": {
       "optional": true
+    },
+    "nodemailer": {
+      "optional": true
     }
   },
   "devDependencies": {
     "@types/nodemailer": "^7.0.11",
-    "playwright": "^1.58.2",
     "vitest": "^4.1.0"
   }
 }

package/templates/grafana/dashboards/agent-workflow.json

@@ -0,0 +1,207 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Avg Steps per Query",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_steps_per_query_sum) / sum(gen_ai_agent_steps_per_query_count) or vector(0)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 1,
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 10 },
+              { "color": "red", "value": 20 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Agent Queries",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_steps_per_query_count)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Total Tool Invocations",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(gen_ai_agent_tool_usage_total)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Unique Tools Used",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "count(count by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short" },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Agent Step Type Breakdown",
+      "description": "Distribution of think / act / observe / answer steps",
+      "type": "piechart",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_agent_step_type) (gen_ai_agent_steps_per_query_sum)",
+          "legendFormat": "{{gen_ai_agent_step_type}}",
+          "refId": "A"
+        }
+      ],
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "values": ["value", "percent"]
+        },
+        "pieType": "donut"
+      }
+    },
+    {
+      "title": "Tool Usage Frequency",
+      "description": "Invocations per tool name",
+      "type": "barchart",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sort_desc(sum by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
+          "legendFormat": "{{gen_ai_agent_tool_name}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "title": "Steps per Query Over Time",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(gen_ai_agent_steps_per_query_sum[5m])) / sum(rate(gen_ai_agent_steps_per_query_count[5m]))",
+          "legendFormat": "avg steps/query",
+          "refId": "avg"
+        },
+        {
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_agent_steps_per_query_bucket[5m])))",
+          "legendFormat": "p95 steps/query",
+          "refId": "p95"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": { "fillOpacity": 10 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Tool Invocations Over Time",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_agent_tool_name) (rate(gen_ai_agent_tool_usage_total[5m]))",
+          "legendFormat": "{{gen_ai_agent_tool_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Steps Distribution Histogram",
+      "description": "How many steps do agent queries typically take?",
+      "type": "histogram",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (le) (gen_ai_agent_steps_per_query_bucket)",
+          "format": "heatmap",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "fillOpacity": 80 }
+        },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["toad-eye", "agent"],
+  "templating": {
+    "list": [
+      {
+        "name": "tool",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": "label_values(gen_ai_agent_tool_usage_total, gen_ai_agent_tool_name)",
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" },
+        "regex": ""
+      }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "utc",
+  "title": "toad-eye / Agent Workflow",
+  "uid": "toad-eye-agent-workflow",
+  "version": 1
+}

package/templates/grafana/dashboards/provider-health.json

@@ -0,0 +1,191 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Provider Status",
+      "description": "healthy (< 10% errors) / degraded (10-50%) / down (> 50%)",
+      "type": "stat",
+      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 0.5 }
+            ]
+          },
+          "mappings": [
+            {
+              "type": "range",
+              "options": {
+                "from": 0,
+                "to": 0.1,
+                "result": { "text": "HEALTHY", "color": "green" }
+              }
+            },
+            {
+              "type": "range",
+              "options": {
+                "from": 0.1,
+                "to": 0.5,
+                "result": { "text": "DEGRADED", "color": "yellow" }
+              }
+            },
+            {
+              "type": "range",
+              "options": {
+                "from": 0.5,
+                "to": 1,
+                "result": { "text": "DOWN", "color": "red" }
+              }
+            }
+          ]
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Request Volume by Provider",
+      "type": "timeseries",
+      "gridPos": { "h": 6, "w": 16, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 15 }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Error Rate by Provider (5m window)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}} error rate",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "custom": { "fillOpacity": 10 },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.1 },
+              { "color": "red", "value": 0.5 }
+            ]
+          }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Error Breakdown by Type",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_provider_name, error_type) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}} — {{error_type}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Provider Availability (24h / 7d / 30d)",
+      "description": "Percentage of time with error rate below 10%",
+      "type": "table",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "1 - (sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total[24h])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total[24h])))",
+          "format": "table",
+          "instant": true,
+          "refId": "uptime_24h"
+        },
+        {
+          "expr": "1 - (sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total[7d])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total[7d])))",
+          "format": "table",
+          "instant": true,
+          "refId": "uptime_7d"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "renameByName": {
+              "gen_ai_provider_name": "Provider",
+              "Value #uptime_24h": "Uptime 24h",
+              "Value #uptime_7d": "Uptime 7d"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "percentunit", "decimals": 2 },
+        "overrides": []
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["toad-eye", "provider-health"],
+  "templating": {
+    "list": [
+      {
+        "name": "provider",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
+        "includeAll": true,
+        "multi": true,
+        "current": { "text": "All", "value": "$__all" },
+        "regex": ""
+      }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "utc",
+  "title": "toad-eye / Provider Health",
+  "uid": "toad-eye-provider-health",
+  "version": 1
+}

package/templates/otel-collector.yml

@@ -8,6 +8,30 @@ processors:
   batch:
     timeout: 1s
 
+  # Tail-based sampling: keeps 100% errors + slow traces, samples healthy traffic.
+  # Only applies to traces pipeline — metrics always get 100%.
+  # To disable: remove tail_sampling from traces pipeline processors.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 100000
+    expected_new_traces_per_sec: 100
+    policies:
+      # Keep all traces with errors
+      - name: errors-always
+        type: status_code
+        status_code:
+          status_codes: [ERROR]
+      # Keep all slow traces (> 2s)
+      - name: slow-traces
+        type: latency
+        latency:
+          threshold_ms: 2000
+      # Sample 10% of remaining healthy traffic
+      - name: probabilistic-fallback
+        type: probabilistic
+        probabilistic:
+          sampling_percentage: 10
+
 exporters:
   otlphttp/jaeger:
     endpoint: http://jaeger:4318
@@ -21,7 +45,7 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [batch]
+      processors: [tail_sampling, batch]
       exporters: [otlphttp/jaeger]
     metrics:
       receivers: [otlp]