toad-eye 2.4.5 → 2.6.0

package/dist/mcp/spans.js

@@ -0,0 +1,67 @@
+/**
+ * OTel span creation for MCP operations.
+ *
+ * Maps MCP methods to OTel GenAI semantic conventions:
+ * - tools/call              → execute_tool {tool_name}
+ * - resources/read          → retrieval {uri}
+ * - prompts/get             → prompt {name}
+ * - sampling/createMessage  → chat {model}
+ */
+import { trace, SpanStatusCode, SpanKind, } from "@opentelemetry/api";
+const TRACER_NAME = "toad-eye-mcp";
+const tracer = trace.getTracer(TRACER_NAME);
+export function startToolSpan(toolName, options) {
+    return tracer.startSpan(`execute_tool ${toolName}`, {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+            "gen_ai.operation.name": "execute_tool",
+            "gen_ai.tool.name": toolName,
+            "mcp.server.name": options.serverName,
+            "mcp.server.version": options.serverVersion,
+        },
+    }, options.parentContext);
+}
+export function startResourceSpan(uri, options) {
+    return tracer.startSpan(`retrieval ${uri}`, {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+            "gen_ai.operation.name": "retrieval",
+            "gen_ai.data_source.id": uri,
+            "mcp.server.name": options.serverName,
+            "mcp.server.version": options.serverVersion,
+        },
+    }, options.parentContext);
+}
+export function startPromptSpan(promptName, options) {
+    return tracer.startSpan(`prompt ${promptName}`, {
+        kind: SpanKind.INTERNAL,
+        attributes: {
+            "gen_ai.operation.name": "prompt",
+            "gen_ai.prompt.name": promptName,
+            "mcp.server.name": options.serverName,
+            "mcp.server.version": options.serverVersion,
+        },
+    }, options.parentContext);
+}
+export function startSamplingSpan(model, options) {
+    return tracer.startSpan(`chat ${model}`, {
+        kind: SpanKind.CLIENT,
+        attributes: {
+            "gen_ai.operation.name": "chat",
+            "gen_ai.request.model": model,
+            "mcp.server.name": options.serverName,
+            "mcp.server.version": options.serverVersion,
+        },
+    }, options.parentContext);
+}
+export function endSpanSuccess(span) {
+    span.setStatus({ code: SpanStatusCode.OK });
+    span.end();
+}
+export function endSpanError(span, error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const errorType = error instanceof Error ? error.constructor.name : "UnknownError";
+    span.setStatus({ code: SpanStatusCode.ERROR, message });
+    span.setAttribute("error.type", errorType);
+    span.end();
+}

package/dist/mcp/types.d.ts

@@ -0,0 +1,34 @@
+import type { Span } from "@opentelemetry/api";
+/** Configuration for toadEyeMiddleware(). */
+export interface ToadMcpOptions {
+    /** Service name for OTel resource. Defaults to MCP server name. */
+    readonly serviceName?: string | undefined;
+    /** OTLP endpoint. Uses existing OTel config if initObservability() was called. */
+    readonly endpoint?: string | undefined;
+    /** Record tool call arguments in spans. Default: false (privacy). */
+    readonly recordInputs?: boolean | undefined;
+    /** Record tool call results in spans. Default: false (privacy). */
+    readonly recordOutputs?: boolean | undefined;
+    /** Keys to redact from recorded arguments. */
+    readonly redactKeys?: readonly string[] | undefined;
+    /** Max payload size in bytes before truncation. Default: 4096. */
+    readonly maxPayloadSize?: number | undefined;
+    /** Extract W3C traceparent from _meta field. Default: true. */
+    readonly propagateContext?: boolean | undefined;
+    /** Hook called on each tool call span. Add custom attributes here. */
+    readonly onToolCall?: ((span: Span, toolName: string, args: unknown) => void) | undefined;
+    /** Hook called on each resource read span. */
+    readonly onResourceRead?: ((span: Span, uri: string) => void) | undefined;
+}
+/** OTel attributes set on MCP spans. */
+export interface McpSpanAttributes {
+    readonly "gen_ai.operation.name": string;
+    readonly "gen_ai.tool.name"?: string;
+    readonly "gen_ai.tool.call.id"?: string;
+    readonly "gen_ai.data_source.id"?: string;
+    readonly "gen_ai.prompt.name"?: string;
+    readonly "mcp.server.name": string;
+    readonly "mcp.server.version": string;
+    readonly "mcp.transport"?: "stdio" | "sse" | "streamable-http";
+    readonly "mcp.session.id"?: string;
+}

package/dist/mcp/types.js

@@ -0,0 +1 @@
+export {};

package/dist/types/attributes.d.ts

@@ -47,6 +47,7 @@ export declare const GEN_AI_ATTRS: {
     readonly COMPLETION: "gen_ai.toad_eye.completion";
     readonly COST: "gen_ai.toad_eye.cost";
     readonly STATUS: "gen_ai.toad_eye.status";
+    readonly CONTEXT_UTILIZATION: "gen_ai.toad_eye.context_utilization";
     readonly SESSION_ID: "session.id";
     readonly TEAM: "toad_eye.team";
     readonly USER_ID: "toad_eye.user_id";

package/dist/types/attributes.js

@@ -53,6 +53,7 @@ export const GEN_AI_ATTRS = {
     COMPLETION: "gen_ai.toad_eye.completion",
     COST: "gen_ai.toad_eye.cost",
     STATUS: "gen_ai.toad_eye.status",
+    CONTEXT_UTILIZATION: "gen_ai.toad_eye.context_utilization",
     SESSION_ID: "session.id",
     // FinOps attribution
     TEAM: "toad_eye.team",

package/dist/types/config.d.ts

@@ -38,6 +38,8 @@ export interface ToadEyeConfig {
     readonly redactDefaults?: boolean | undefined;
     /** Log what was masked to console (for debugging redaction config). No data is sent externally. */
     readonly auditMasking?: boolean | undefined;
+    /** Record content for only this fraction of calls (0.0–1.0). E.g., 0.01 = 1%. Enables debugging without full storage/privacy cost. */
+    readonly contentSamplingRate?: number | undefined;
     /** Configure trace sampling. Tail sampling happens in OTel Collector, not SDK. */
     readonly sampling?: SamplingConfig | undefined;
     readonly instrument?: readonly InstrumentTarget[] | undefined;
@@ -47,6 +49,13 @@ export interface ToadEyeConfig {
     readonly onBudgetExceeded?: BudgetExceededMode | undefined;
     /** Callback for 'downgrade' mode — receives original provider/model, returns replacement. */
     readonly downgradeCallback?: DowngradeCallback | undefined;
+    /** Warn when context window utilization exceeds thresholds (post-call). */
+    readonly contextGuard?: {
+        /** Warn when utilization exceeds this ratio (0.0–1.0). Default: none. */
+        readonly warnAt?: number | undefined;
+        /** Alert (span event + metric + warning) when utilization exceeds this ratio. Default: none. */
+        readonly alertAt?: number | undefined;
+    } | undefined;
     /** Static session ID — all spans will carry this value as `session.id`. */
     readonly sessionId?: string | undefined;
     /** Dynamic session ID extractor — called per traceLLMCall to resolve session ID. */

package/dist/types/metrics.d.ts

@@ -14,6 +14,7 @@ export declare const GEN_AI_METRICS: {
     readonly ERRORS: "gen_ai.client.errors";
     readonly AGENT_STEPS_PER_QUERY: "gen_ai.agent.steps_per_query";
     readonly AGENT_TOOL_USAGE: "gen_ai.agent.tool_usage";
+    readonly AGENT_TOOL_DURATION: "gen_ai.agent.tool_duration";
     readonly GUARD_EVALUATIONS: "gen_ai.toad_eye.guard.evaluations";
     readonly GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block";
     readonly SEMANTIC_DRIFT: "gen_ai.toad_eye.semantic_drift";
@@ -23,6 +24,13 @@ export declare const GEN_AI_METRICS: {
     readonly BUDGET_DOWNGRADED: "gen_ai.toad_eye.budget.downgraded";
     readonly RESPONSE_EMPTY: "gen_ai.toad_eye.response.empty";
     readonly RESPONSE_LATENCY_PER_TOKEN: "gen_ai.toad_eye.response.latency_per_token";
+    readonly CONTEXT_UTILIZATION: "gen_ai.toad_eye.context_utilization";
+    readonly CONTEXT_BLOCKED: "gen_ai.toad_eye.context.blocked";
+    readonly MCP_TOOL_DURATION: "gen_ai.mcp.tool.duration";
+    readonly MCP_TOOL_CALLS: "gen_ai.mcp.tool.calls";
+    readonly MCP_TOOL_ERRORS: "gen_ai.mcp.tool.errors";
+    readonly MCP_RESOURCE_READS: "gen_ai.mcp.resource.reads";
+    readonly MCP_SESSION_ACTIVE: "gen_ai.mcp.session.active";
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export declare const LLM_METRICS: {

package/dist/types/metrics.js

@@ -15,6 +15,7 @@ export const GEN_AI_METRICS = {
     // Agent metrics
     AGENT_STEPS_PER_QUERY: "gen_ai.agent.steps_per_query",
     AGENT_TOOL_USAGE: "gen_ai.agent.tool_usage",
+    AGENT_TOOL_DURATION: "gen_ai.agent.tool_duration",
     // Guard metrics
     GUARD_EVALUATIONS: "gen_ai.toad_eye.guard.evaluations",
     GUARD_WOULD_BLOCK: "gen_ai.toad_eye.guard.would_block",
@@ -29,6 +30,15 @@ export const GEN_AI_METRICS = {
     // Response quality proxy metrics
     RESPONSE_EMPTY: "gen_ai.toad_eye.response.empty",
     RESPONSE_LATENCY_PER_TOKEN: "gen_ai.toad_eye.response.latency_per_token",
+    // Context utilization
+    CONTEXT_UTILIZATION: "gen_ai.toad_eye.context_utilization",
+    CONTEXT_BLOCKED: "gen_ai.toad_eye.context.blocked",
+    // MCP metrics
+    MCP_TOOL_DURATION: "gen_ai.mcp.tool.duration",
+    MCP_TOOL_CALLS: "gen_ai.mcp.tool.calls",
+    MCP_TOOL_ERRORS: "gen_ai.mcp.tool.errors",
+    MCP_RESOURCE_READS: "gen_ai.mcp.resource.reads",
+    MCP_SESSION_ACTIVE: "gen_ai.mcp.session.active",
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export const LLM_METRICS = {

package/dist/types/spans.d.ts

@@ -11,6 +11,10 @@ export interface AgentStepInput {
     readonly toolName?: string | undefined;
     /** Tool type per OTel GenAI spec: function, extension, retrieval, builtin */
     readonly toolType?: "function" | "extension" | "retrieval" | "builtin" | undefined;
+    /** Duration of tool execution in milliseconds (for act steps). */
+    readonly toolDurationMs?: number | undefined;
+    /** Whether the tool execution succeeded or failed (for act steps). */
+    readonly toolStatus?: "success" | "error" | undefined;
     /** Target agent name for handoff steps */
     readonly toAgent?: string | undefined;
     /** Reason for handoff */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toad-eye",
-  "version": "2.4.5",
+  "version": "2.6.0",
   "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
   "type": "module",
   "main": "dist/index.js",
@@ -30,6 +30,11 @@
       "types": "./dist/vercel.d.ts",
       "import": "./dist/vercel.js",
       "default": "./dist/vercel.js"
+    },
+    "./mcp": {
+      "types": "./dist/mcp/index.d.ts",
+      "import": "./dist/mcp/index.js",
+      "default": "./dist/mcp/index.js"
     }
   },
   "bin": {
@@ -87,7 +92,8 @@
     "@anthropic-ai/sdk": ">=0.30.0",
     "@google/generative-ai": ">=0.20.0",
     "nodemailer": ">=8.0.0",
-    "openai": ">=4.0.0"
+    "openai": ">=4.0.0",
+    "@modelcontextprotocol/sdk": ">=1.0.0"
   },
   "peerDependenciesMeta": {
     "openai": {
@@ -101,6 +107,9 @@
     },
     "nodemailer": {
       "optional": true
+    },
+    "@modelcontextprotocol/sdk": {
+      "optional": true
     }
   },
   "devDependencies": {

package/templates/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   collector:
-    image: otel/opentelemetry-collector-contrib:latest
+    image: otel/opentelemetry-collector-contrib:0.102.0
     volumes:
       - ./otel-collector.yml:/etc/otelcol-contrib/config.yaml
     ports:
@@ -10,14 +10,14 @@ services:
       - prometheus
 
   prometheus:
-    image: prom/prometheus:latest
+    image: prom/prometheus:v2.53.0
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
 
   jaeger:
-    image: jaegertracing/all-in-one:latest
+    image: jaegertracing/all-in-one:1.58
     environment:
       - COLLECTOR_OTLP_ENABLED=true
     ports:
@@ -25,7 +25,7 @@ services:
       - "4317:4317"
 
   grafana:
-    image: grafana/grafana:latest
+    image: grafana/grafana:11.1.0
     volumes:
       - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards

package/templates/grafana/dashboards/agent-workflow.json

@@ -1,5 +1,7 @@
 {
-  "annotations": { "list": [] },
+  "annotations": {
+    "list": []
+  },
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
@@ -9,8 +11,16 @@
     {
       "title": "Avg Steps per Query",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(gen_ai_agent_steps_per_query_sum) / sum(gen_ai_agent_steps_per_query_count) or vector(0)",
@@ -23,9 +33,18 @@
           "decimals": 1,
           "thresholds": {
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 10 },
-              { "color": "red", "value": 20 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 10
+              },
+              {
+                "color": "red",
+                "value": 20
+              }
             ]
           }
         },
@@ -35,8 +54,16 @@
     {
       "title": "Total Agent Queries",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(gen_ai_agent_steps_per_query_count)",
@@ -44,15 +71,25 @@
         }
       ],
       "fieldConfig": {
-        "defaults": { "unit": "short" },
+        "defaults": {
+          "unit": "short"
+        },
         "overrides": []
       }
     },
     {
       "title": "Total Tool Invocations",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(gen_ai_agent_tool_usage_total)",
@@ -60,15 +97,25 @@
         }
       ],
       "fieldConfig": {
-        "defaults": { "unit": "short" },
+        "defaults": {
+          "unit": "short"
+        },
         "overrides": []
       }
     },
     {
       "title": "Unique Tools Used",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "count(count by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
@@ -76,7 +123,9 @@
         }
       ],
       "fieldConfig": {
-        "defaults": { "unit": "short" },
+        "defaults": {
+          "unit": "short"
+        },
         "overrides": []
       }
     },
@@ -84,8 +133,16 @@
       "title": "Agent Step Type Breakdown",
       "description": "Distribution of think / act / observe / answer steps",
       "type": "piechart",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum by (gen_ai_agent_step_type) (gen_ai_agent_steps_per_query_sum)",
@@ -106,8 +163,16 @@
       "title": "Tool Usage Frequency",
       "description": "Invocations per tool name",
       "type": "barchart",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sort_desc(sum by (gen_ai_agent_tool_name) (gen_ai_agent_tool_usage_total))",
@@ -119,8 +184,16 @@
     {
       "title": "Steps per Query Over Time",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(rate(gen_ai_agent_steps_per_query_sum[5m])) / sum(rate(gen_ai_agent_steps_per_query_count[5m]))",
@@ -136,7 +209,9 @@
       "fieldConfig": {
         "defaults": {
           "unit": "short",
-          "custom": { "fillOpacity": 10 }
+          "custom": {
+            "fillOpacity": 10
+          }
         },
         "overrides": []
       }
@@ -144,8 +219,16 @@
     {
       "title": "Tool Invocations Over Time",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum by (gen_ai_agent_tool_name) (rate(gen_ai_agent_tool_usage_total[5m]))",
@@ -156,7 +239,12 @@
       "fieldConfig": {
         "defaults": {
           "unit": "reqps",
-          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+          "custom": {
+            "fillOpacity": 20,
+            "stacking": {
+              "mode": "normal"
+            }
+          }
         },
         "overrides": []
       }
@@ -165,8 +253,16 @@
       "title": "Steps Distribution Histogram",
       "description": "How many steps do agent queries typically take?",
       "type": "histogram",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum by (le) (gen_ai_agent_steps_per_query_bucket)",
@@ -176,7 +272,9 @@
       ],
       "fieldConfig": {
         "defaults": {
-          "custom": { "fillOpacity": 80 }
+          "custom": {
+            "fillOpacity": 80
+          }
         },
         "overrides": []
       }
@@ -186,19 +284,44 @@
   "tags": ["toad-eye", "agent"],
   "templating": {
     "list": [
+      {
+        "name": "service",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": "label_values(gen_ai_client_requests_total, job)",
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true
+      },
       {
         "name": "tool",
         "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
         "query": "label_values(gen_ai_agent_tool_usage_total, gen_ai_agent_tool_name)",
         "includeAll": true,
         "multi": true,
-        "current": { "text": "All", "value": "$__all" },
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
         "regex": ""
       }
     ]
   },
-  "time": { "from": "now-6h", "to": "now" },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
   "timepicker": {},
   "timezone": "utc",
   "title": "toad-eye / Agent Workflow",