toad-eye 2.6.1 → 2.8.0

package/dist/mcp/spans.js

@@ -1,56 +1,71 @@
 /**
  * OTel span creation for MCP operations.
  *
- * Maps MCP methods to OTel GenAI semantic conventions:
- * - tools/call              → execute_tool {tool_name}
- * - resources/read          → retrieval {uri}
- * - prompts/get             → prompt {name}
- * - sampling/createMessage  → chat {model}
+ * Follows OTel MCP semantic conventions (PR #2083):
+ * - tools/call              → `tools/call {tool_name}`
+ * - resources/read          → `resources/read {uri}`
+ * - prompts/get             → `prompts/get {name}`
+ * - sampling/createMessage  → `sampling/createMessage {model}`
  */
 import { trace, SpanStatusCode, SpanKind, } from "@opentelemetry/api";
 const TRACER_NAME = "toad-eye-mcp";
 const tracer = trace.getTracer(TRACER_NAME);
+/** MCP method names per OTel semconv. */
+export const MCP_METHODS = {
+    TOOLS_CALL: "tools/call",
+    RESOURCES_READ: "resources/read",
+    PROMPTS_GET: "prompts/get",
+    SAMPLING_CREATE_MESSAGE: "sampling/createMessage",
+};
+/** Build common MCP attributes shared across all span types. */
+function baseAttrs(method, options) {
+    const attrs = {
+        "gen_ai.operation.name": method,
+        "mcp.method.name": method,
+        "mcp.server.name": options.serverName,
+        "mcp.server.version": options.serverVersion,
+    };
+    if (options.sessionId)
+        attrs["mcp.session.id"] = options.sessionId;
+    if (options.protocolVersion)
+        attrs["mcp.protocol.version"] = options.protocolVersion;
+    if (options.networkTransport)
+        attrs["network.transport"] = options.networkTransport;
+    return attrs;
+}
 export function startToolSpan(toolName, options) {
-    return tracer.startSpan(`execute_tool ${toolName}`, {
-        kind: SpanKind.INTERNAL,
+    return tracer.startSpan(`${MCP_METHODS.TOOLS_CALL} ${toolName}`, {
+        kind: SpanKind.SERVER,
         attributes: {
-            "gen_ai.operation.name": "execute_tool",
+            ...baseAttrs(MCP_METHODS.TOOLS_CALL, options),
             "gen_ai.tool.name": toolName,
-            "mcp.server.name": options.serverName,
-            "mcp.server.version": options.serverVersion,
         },
     }, options.parentContext);
 }
 export function startResourceSpan(uri, options) {
-    return tracer.startSpan(`retrieval ${uri}`, {
-        kind: SpanKind.INTERNAL,
+    return tracer.startSpan(`${MCP_METHODS.RESOURCES_READ} ${uri}`, {
+        kind: SpanKind.SERVER,
         attributes: {
-            "gen_ai.operation.name": "retrieval",
+            ...baseAttrs(MCP_METHODS.RESOURCES_READ, options),
             "gen_ai.data_source.id": uri,
-            "mcp.server.name": options.serverName,
-            "mcp.server.version": options.serverVersion,
         },
     }, options.parentContext);
 }
 export function startPromptSpan(promptName, options) {
-    return tracer.startSpan(`prompt ${promptName}`, {
-        kind: SpanKind.INTERNAL,
+    return tracer.startSpan(`${MCP_METHODS.PROMPTS_GET} ${promptName}`, {
+        kind: SpanKind.SERVER,
         attributes: {
-            "gen_ai.operation.name": "prompt",
+            ...baseAttrs(MCP_METHODS.PROMPTS_GET, options),
             "gen_ai.prompt.name": promptName,
-            "mcp.server.name": options.serverName,
-            "mcp.server.version": options.serverVersion,
         },
     }, options.parentContext);
 }
 export function startSamplingSpan(model, options) {
-    return tracer.startSpan(`chat ${model}`, {
+    return tracer.startSpan(`${MCP_METHODS.SAMPLING_CREATE_MESSAGE} ${model}`, {
         kind: SpanKind.CLIENT,
         attributes: {
-            "gen_ai.operation.name": "chat",
+            ...baseAttrs(MCP_METHODS.SAMPLING_CREATE_MESSAGE, options),
             "gen_ai.request.model": model,
-            "mcp.server.name": options.serverName,
-            "mcp.server.version": options.serverVersion,
         },
     }, options.parentContext);
 }
@@ -63,5 +78,10 @@ export function endSpanError(span, error) {
     const errorType = error instanceof Error ? error.constructor.name : "UnknownError";
     span.setStatus({ code: SpanStatusCode.ERROR, message });
     span.setAttribute("error.type", errorType);
+    // JSON-RPC error code if available (MCP SDK errors often carry a code)
+    const code = error?.code;
+    if (typeof code === "number") {
+        span.setAttribute("rpc.response.status_code", code);
+    }
     span.end();
 }

package/dist/mcp/types.d.ts

@@ -1,18 +1,16 @@
 import type { Span } from "@opentelemetry/api";
 /** Configuration for toadEyeMiddleware(). */
 export interface ToadMcpOptions {
-    /** Service name for OTel resource. Defaults to MCP server name. */
-    readonly serviceName?: string | undefined;
-    /** OTLP endpoint. Uses existing OTel config if initObservability() was called. */
-    readonly endpoint?: string | undefined;
     /** Record tool call arguments in spans. Default: false (privacy). */
     readonly recordInputs?: boolean | undefined;
     /** Record tool call results in spans. Default: false (privacy). */
     readonly recordOutputs?: boolean | undefined;
-    /** Keys to redact from recorded arguments. */
+    /** Keys to redact from recorded arguments (recursive). */
     readonly redactKeys?: readonly string[] | undefined;
-    /** Max payload size in bytes before truncation. Default: 4096. */
+    /** Max payload size in characters before truncation. Default: 4096. */
     readonly maxPayloadSize?: number | undefined;
+    /** Explicit session ID. Auto-generated if not provided. */
+    readonly sessionId?: string | undefined;
     /** Extract W3C traceparent from _meta field. Default: true. */
     readonly propagateContext?: boolean | undefined;
     /** Hook called on each tool call span. Add custom attributes here. */
@@ -20,15 +18,3 @@ export interface ToadMcpOptions {
     /** Hook called on each resource read span. */
     readonly onResourceRead?: ((span: Span, uri: string) => void) | undefined;
 }
-/** OTel attributes set on MCP spans. */
-export interface McpSpanAttributes {
-    readonly "gen_ai.operation.name": string;
-    readonly "gen_ai.tool.name"?: string;
-    readonly "gen_ai.tool.call.id"?: string;
-    readonly "gen_ai.data_source.id"?: string;
-    readonly "gen_ai.prompt.name"?: string;
-    readonly "mcp.server.name": string;
-    readonly "mcp.server.version": string;
-    readonly "mcp.transport"?: "stdio" | "sse" | "streamable-http";
-    readonly "mcp.session.id"?: string;
-}

package/dist/types/metrics.d.ts

@@ -31,6 +31,7 @@ export declare const GEN_AI_METRICS: {
     readonly MCP_TOOL_ERRORS: "gen_ai.mcp.tool.errors";
     readonly MCP_RESOURCE_READS: "gen_ai.mcp.resource.reads";
     readonly MCP_SESSION_ACTIVE: "gen_ai.mcp.session.active";
+    readonly MCP_TOOL_CALLERS: "gen_ai.mcp.tool.callers";
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export declare const LLM_METRICS: {

package/dist/types/metrics.js

@@ -39,6 +39,7 @@ export const GEN_AI_METRICS = {
     MCP_TOOL_ERRORS: "gen_ai.mcp.tool.errors",
     MCP_RESOURCE_READS: "gen_ai.mcp.resource.reads",
     MCP_SESSION_ACTIVE: "gen_ai.mcp.session.active",
+    MCP_TOOL_CALLERS: "gen_ai.mcp.tool.callers",
 };
 /** @deprecated Use GEN_AI_METRICS instead. Kept for backward compatibility. */
 export const LLM_METRICS = {

package/dist/types/providers.d.ts

@@ -6,5 +6,7 @@ export type LLMProvider = "anthropic" | "gemini" | "openai" | (string & {});
 /**
  * All instrumentable targets — includes LLM providers + framework SDKs.
  * 'ai' = Vercel AI SDK (uses SpanProcessor, not monkey-patching).
+ * 'mcp' = MCP SDK McpServer (patches prototype for auto-instrumentation).
+ * 'mcp-client' = MCP SDK Client (patches callTool/readResource/getPrompt for distributed tracing).
  */
-export type InstrumentTarget = LLMProvider | "ai";
+export type InstrumentTarget = LLMProvider | "ai" | "mcp" | "mcp-client";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "toad-eye",
-  "version": "2.6.1",
-  "description": "Auto-instrumentation, cost tracking, budget guards, and observability for LLM services",
+  "version": "2.8.0",
+  "description": "Observability for MCP servers and LLM applications — traces, metrics, dashboards. One-line instrumentation for OpenAI, Anthropic, Gemini, and MCP. Self-hosted with Grafana + Prometheus + Jaeger.",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -49,7 +49,7 @@
     "build": "tsc -p tsconfig.build.json",
     "test": "vitest run",
     "test:watch": "vitest",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "test -s README.md || { echo '❌ README.md is missing or empty'; exit 1; } && npm run build"
   },
   "repository": {
     "type": "git",
@@ -58,6 +58,11 @@
   "homepage": "https://github.com/vola-trebla/toad-eye#readme",
   "author": "albertalov",
   "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "mcp-server",
+    "mcp-observability",
+    "mcp-monitoring",
     "opentelemetry",
     "llm",
     "instrumentation",
@@ -71,9 +76,13 @@
     "cost-tracking",
     "auto-instrumentation",
     "gen-ai",
+    "ai-agent",
+    "tool-calling",
     "budget-guards",
     "finops",
-    "streaming"
+    "streaming",
+    "jaeger",
+    "prometheus"
   ],
   "license": "ISC",
   "dependencies": {

package/templates/alerts.example.yml

@@ -52,3 +52,24 @@ alerts:
     condition: ratio_5m > 0.2
     channels: [slack]
     cooldown: 15
+
+  # --- MCP Server alerts ---
+
+  # MCP tool error rate: >10% over 5 minutes
+  - name: mcp_tool_error_rate
+    metric: gen_ai.mcp.tool.errors
+    condition: ratio_5m > 0.1
+    channels: [slack]
+
+  # MCP tool latency spike: avg > 2 seconds over 5 minutes
+  - name: mcp_tool_latency
+    metric: gen_ai.mcp.tool.duration
+    condition: avg_5m > 2000
+    channels: [slack]
+
+  # MCP tool call spike: >100 calls in 5 minutes
+  - name: mcp_tool_call_spike
+    metric: gen_ai.mcp.tool.calls
+    condition: sum_5m > 100
+    channels: [slack]
+    cooldown: 15

package/templates/grafana/dashboards/mcp-analytics.json

@@ -0,0 +1,295 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Tool Popularity Ranking",
+      "description": "Most called tools by total invocations",
+      "type": "bargauge",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "topk(10, sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_calls_total[1h])))",
+          "legendFormat": "{{gen_ai_tool_name}}",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "steps": [
+              { "color": "blue", "value": null },
+              { "color": "green", "value": 10 },
+              { "color": "yellow", "value": 100 },
+              { "color": "red", "value": 500 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "showUnfilled": true,
+        "minVizWidth": 0,
+        "minVizHeight": 10
+      }
+    },
+    {
+      "title": "Unique Callers per Tool",
+      "description": "Number of distinct sessions calling each tool",
+      "type": "bargauge",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "count by (gen_ai_tool_name) (sum by (gen_ai_tool_name, mcp_session_id) (gen_ai_mcp_tool_callers_total))",
+          "legendFormat": "{{gen_ai_tool_name}}",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "steps": [
+              { "color": "purple", "value": null },
+              { "color": "blue", "value": 5 },
+              { "color": "green", "value": 20 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "orientation": "horizontal",
+        "displayMode": "gradient",
+        "showUnfilled": true,
+        "minVizWidth": 0,
+        "minVizHeight": 10
+      }
+    },
+    {
+      "title": "Tool Usage Over Time",
+      "description": "Call rate per tool — spot trends and spikes",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_calls_total[5m]))",
+          "legendFormat": "{{gen_ai_tool_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 20,
+            "pointSize": 5,
+            "showPoints": "auto",
+            "stacking": { "mode": "none" }
+          },
+          "color": { "mode": "palette-classic" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": ["mean", "max", "sum"]
+        },
+        "tooltip": { "mode": "multi" }
+      }
+    },
+    {
+      "title": "Cost per LLM Model (via sampling)",
+      "description": "LLM cost breakdown — visible when MCP tools call LLMs internally",
+      "type": "piechart",
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_request_model) (increase(gen_ai_client_request_cost_USD_sum[1h]))",
+          "legendFormat": "{{gen_ai_request_model}}",
+          "refId": "A",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 4,
+          "color": { "mode": "palette-classic" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "values": ["value", "percent"]
+        },
+        "pieType": "donut",
+        "tooltip": { "mode": "multi" }
+      }
+    },
+    {
+      "title": "Cost Over Time",
+      "description": "LLM cost rate — attribute to tools via trace context",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 16, "x": 8, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_request_cost_USD_sum[5m])) * 3600",
+          "legendFormat": "{{gen_ai_request_model}} ($/hr)",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD",
+          "decimals": 4,
+          "custom": {
+            "drawStyle": "line",
+            "lineInterpolation": "smooth",
+            "fillOpacity": 15,
+            "pointSize": 5,
+            "showPoints": "auto"
+          },
+          "color": { "mode": "palette-classic" }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "bottom",
+          "calcs": ["sum", "mean"]
+        },
+        "tooltip": { "mode": "multi" }
+      }
+    },
+    {
+      "title": "Tool Performance Summary",
+      "description": "All tools: call count, avg duration, error rate, unique callers",
+      "type": "table",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_calls_total[1h]))",
+          "legendFormat": "",
+          "refId": "calls",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_sum[1h])) / sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_count[1h]))",
+          "legendFormat": "",
+          "refId": "avg_duration",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_errors_total[1h])) / sum by (gen_ai_tool_name) (increase(gen_ai_mcp_tool_calls_total[1h]))",
+          "legendFormat": "",
+          "refId": "error_rate",
+          "format": "table",
+          "instant": true
+        },
+        {
+          "expr": "count by (gen_ai_tool_name) (sum by (gen_ai_tool_name, mcp_session_id) (gen_ai_mcp_tool_callers_total))",
+          "legendFormat": "",
+          "refId": "callers",
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "transformations": [
+        { "id": "merge" },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true },
+            "renameByName": {
+              "gen_ai_tool_name": "Tool",
+              "Value #calls": "Calls (1h)",
+              "Value #avg_duration": "Avg Duration (ms)",
+              "Value #error_rate": "Error Rate",
+              "Value #callers": "Unique Callers"
+            }
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": { "align": "center" }
+        },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Avg Duration (ms)" },
+            "properties": [
+              { "id": "unit", "value": "ms" },
+              { "id": "decimals", "value": 1 }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Error Rate" },
+            "properties": [
+              { "id": "unit", "value": "percentunit" },
+              { "id": "decimals", "value": 2 },
+              {
+                "id": "thresholds",
+                "value": {
+                  "mode": "absolute",
+                  "steps": [
+                    { "color": "green", "value": null },
+                    { "color": "yellow", "value": 0.05 },
+                    { "color": "red", "value": 0.1 }
+                  ]
+                }
+              },
+              { "id": "custom.displayMode", "value": "color-background" }
+            ]
+          },
+          {
+            "matcher": { "id": "byName", "options": "Calls (1h)" },
+            "properties": [{ "id": "decimals", "value": 0 }]
+          }
+        ]
+      }
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["toad-eye", "mcp", "analytics", "cost"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "MCP Tool Analytics",
+  "uid": "toad-eye-mcp-analytics",
+  "version": 1
+}