toad-eye 2.9.2 → 2.11.0

package/templates/grafana/dashboards/mcp-e2e.json

@@ -12,8 +12,16 @@
       "title": "Total Request Rate",
       "description": "Combined rate of MCP client + server operations",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(rate(gen_ai_mcp_tool_calls_total[5m])) + sum(rate(gen_ai_client_requests_total[5m])) or vector(0)",
@@ -26,11 +34,21 @@
           "decimals": 2,
           "thresholds": {
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 50 },
-              { "color": "red", "value": 100 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 50
+              },
+              {
+                "color": "red",
+                "value": 100
+              }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -39,11 +57,19 @@
       "title": "LLM Avg Latency",
       "description": "Average LLM call duration (client-side)",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_operation_duration_sum[5m])) / sum(rate(gen_ai_client_operation_duration_count[5m])) or vector(0)",
+          "expr": "sum(rate(gen_ai_client_operation_duration_milliseconds_sum[5m])) / sum(rate(gen_ai_client_operation_duration_milliseconds_count[5m])) or vector(0)",
           "refId": "A"
         }
       ],
@@ -53,11 +79,21 @@
           "decimals": 0,
           "thresholds": {
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 1000 },
-              { "color": "red", "value": 5000 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1000
+              },
+              {
+                "color": "red",
+                "value": 5000
+              }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -66,8 +102,16 @@
       "title": "Tool Avg Latency",
       "description": "Average MCP tool execution duration (server-side)",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(rate(gen_ai_mcp_tool_duration_sum[5m])) / sum(rate(gen_ai_mcp_tool_duration_count[5m])) or vector(0)",
@@ -80,11 +124,21 @@
           "decimals": 0,
           "thresholds": {
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 500 },
-              { "color": "red", "value": 2000 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 500
+              },
+              {
+                "color": "red",
+                "value": 2000
+              }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -93,8 +147,16 @@
       "title": "End-to-End Error Rate",
       "description": "Combined error rate across LLM + MCP",
       "type": "stat",
-      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "(sum(rate(gen_ai_client_errors_total[5m])) + sum(rate(gen_ai_mcp_tool_errors_total[5m]))) / (sum(rate(gen_ai_client_requests_total[5m])) + sum(rate(gen_ai_mcp_tool_calls_total[5m]))) or vector(0)",
@@ -107,11 +169,21 @@
           "decimals": 2,
           "thresholds": {
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 0.05 },
-              { "color": "red", "value": 0.1 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.05
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -120,11 +192,19 @@
       "title": "Latency Breakdown: LLM vs Tool",
       "description": "Compare LLM call latency vs MCP tool execution latency over time",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_operation_duration_sum[5m])) / sum(rate(gen_ai_client_operation_duration_count[5m]))",
+          "expr": "sum(rate(gen_ai_client_operation_duration_milliseconds_sum[5m])) / sum(rate(gen_ai_client_operation_duration_milliseconds_count[5m]))",
           "legendFormat": "LLM Avg Latency",
           "refId": "A"
         },
@@ -143,24 +223,37 @@
             "fillOpacity": 20,
             "pointSize": 5,
             "showPoints": "auto"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": [
           {
-            "matcher": { "id": "byName", "options": "LLM Avg Latency" },
+            "matcher": {
+              "id": "byName",
+              "options": "LLM Avg Latency"
+            },
             "properties": [
               {
                 "id": "color",
-                "value": { "fixedColor": "orange", "mode": "fixed" }
+                "value": {
+                  "fixedColor": "orange",
+                  "mode": "fixed"
+                }
               }
             ]
           },
           {
-            "matcher": { "id": "byName", "options": "Tool Avg Latency" },
+            "matcher": {
+              "id": "byName",
+              "options": "Tool Avg Latency"
+            },
             "properties": [
               {
                 "id": "color",
-                "value": { "fixedColor": "blue", "mode": "fixed" }
+                "value": {
+                  "fixedColor": "blue",
+                  "mode": "fixed"
+                }
               }
             ]
           }
@@ -172,15 +265,25 @@
           "placement": "bottom",
           "calcs": ["mean", "max"]
         },
-        "tooltip": { "mode": "multi" }
+        "tooltip": {
+          "mode": "multi"
+        }
       }
     },
     {
       "title": "Tool Call Waterfall by Tool",
       "description": "Duration per tool over time — identify slow tools",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_sum[5m])) / sum by (gen_ai_tool_name) (rate(gen_ai_mcp_tool_duration_count[5m]))",
@@ -194,9 +297,14 @@
           "custom": {
             "drawStyle": "bars",
             "fillOpacity": 60,
-            "stacking": { "mode": "normal" }
+            "stacking": {
+              "mode": "normal"
+            }
+          },
+          "color": {
+            "mode": "palette-classic"
           },
-          "color": { "mode": "palette-classic" }
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -206,15 +314,25 @@
           "placement": "bottom",
           "calcs": ["mean", "sum"]
         },
-        "tooltip": { "mode": "multi" }
+        "tooltip": {
+          "mode": "multi"
+        }
       }
     },
     {
       "title": "Error Propagation: LLM vs Tool",
       "description": "Error rates split by LLM errors and MCP tool errors",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum(rate(gen_ai_client_errors_total[5m]))",
@@ -236,24 +354,37 @@
             "fillOpacity": 30,
             "pointSize": 5,
             "showPoints": "auto"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": [
           {
-            "matcher": { "id": "byName", "options": "LLM Errors" },
+            "matcher": {
+              "id": "byName",
+              "options": "LLM Errors"
+            },
             "properties": [
               {
                 "id": "color",
-                "value": { "fixedColor": "red", "mode": "fixed" }
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
               }
             ]
           },
           {
-            "matcher": { "id": "byName", "options": "Tool Errors" },
+            "matcher": {
+              "id": "byName",
+              "options": "Tool Errors"
+            },
             "properties": [
               {
                 "id": "color",
-                "value": { "fixedColor": "purple", "mode": "fixed" }
+                "value": {
+                  "fixedColor": "purple",
+                  "mode": "fixed"
+                }
               }
             ]
           }
@@ -265,15 +396,25 @@
           "placement": "bottom",
           "calcs": ["sum"]
         },
-        "tooltip": { "mode": "multi" }
+        "tooltip": {
+          "mode": "multi"
+        }
       }
     },
     {
       "title": "Cost: LLM Calls",
       "description": "Total LLM cost over time — visible when tools call LLMs via sampling",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_request_cost_USD_sum[5m]))",
@@ -292,7 +433,10 @@
             "pointSize": 5,
             "showPoints": "auto"
           },
-          "color": { "mode": "palette-classic" }
+          "color": {
+            "mode": "palette-classic"
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -302,7 +446,9 @@
           "placement": "bottom",
           "calcs": ["sum"]
         },
-        "tooltip": { "mode": "multi" }
+        "tooltip": {
+          "mode": "multi"
+        }
       }
     }
   ],
@@ -315,6 +461,7 @@
     "from": "now-1h",
     "to": "now"
   },
+  "refresh": "30s",
   "timepicker": {},
   "timezone": "browser",
   "title": "MCP End-to-End",

package/templates/grafana/dashboards/mcp-server.json

@@ -46,7 +46,8 @@
                 "value": 100
               }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -89,7 +90,8 @@
                 "value": 2000
               }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -132,7 +134,8 @@
                 "value": 0.1
               }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -167,7 +170,36 @@
                 "value": null
               }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
+        },
+        "overrides": []
+      }
+    },
+    {
+      "title": "Tool Hallucination Rate",
+      "description": "Agent tried to call a non-existent tool (JSON-RPC -32601). Spikes indicate the LLM lost track of available tools.",
+      "type": "stat",
+      "gridPos": { "h": 3, "w": 24, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "targets": [
+        {
+          "expr": "sum(rate(gen_ai_mcp_tool_hallucinations_total[5m])) or vector(0)",
+          "legendFormat": "hallucinations/s",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "thresholds": {
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 0.01 },
+              { "color": "red", "value": 0.1 }
+            ]
+          },
+          "noValue": "No hallucinations detected"
         },
         "overrides": []
       }
@@ -201,7 +233,8 @@
             "fillOpacity": 15,
             "pointSize": 5,
             "showPoints": "auto"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -250,7 +283,8 @@
             "fillOpacity": 10,
             "pointSize": 5,
             "showPoints": "auto"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -297,7 +331,8 @@
           },
           "color": {
             "mode": "palette-classic"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -341,7 +376,8 @@
             "fillOpacity": 15,
             "pointSize": 5,
             "showPoints": "auto"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       },
@@ -423,7 +459,8 @@
         "defaults": {
           "custom": {
             "align": "center"
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": [
           {

package/templates/grafana/dashboards/model-comparison.json

@@ -35,7 +35,8 @@
             "drawStyle": "line",
             "fillOpacity": 15,
             "lineWidth": 2
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -67,7 +68,8 @@
             "drawStyle": "line",
             "fillOpacity": 15,
             "lineWidth": 2
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -99,7 +101,8 @@
             "drawStyle": "line",
             "fillOpacity": 15,
             "lineWidth": 2
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -131,7 +134,8 @@
             "drawStyle": "line",
             "fillOpacity": 15,
             "lineWidth": 2
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -163,7 +167,8 @@
             "drawStyle": "line",
             "fillOpacity": 15,
             "lineWidth": 2
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }

package/templates/grafana/dashboards/provider-health.json

@@ -83,7 +83,8 @@
                 }
               }
             }
-          ]
+          ],
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -113,7 +114,8 @@
           "unit": "reqps",
           "custom": {
             "fillOpacity": 15
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -160,13 +162,14 @@
                 "value": 0.5
               }
             ]
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
     },
     {
-      "title": "Error Breakdown by Type",
+      "title": "Error Breakdown by Model",
       "type": "timeseries",
       "gridPos": {
         "h": 8,
@@ -180,8 +183,8 @@
       },
       "targets": [
         {
-          "expr": "sum by (gen_ai_provider_name, error_type) (rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
-          "legendFormat": "{{gen_ai_provider_name}} — {{error_type}}",
+          "expr": "sum by (gen_ai_provider_name, gen_ai_request_model) (rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "legendFormat": "{{gen_ai_provider_name}} — {{gen_ai_request_model}}",
           "refId": "A"
         }
       ],
@@ -193,7 +196,8 @@
             "stacking": {
               "mode": "normal"
             }
-          }
+          },
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -241,7 +245,8 @@
       "fieldConfig": {
         "defaults": {
           "unit": "percentunit",
-          "decimals": 2
+          "decimals": 2,
+          "noValue": "No data yet — run: npx toad-eye demo"
         },
         "overrides": []
       }
@@ -276,6 +281,7 @@
         },
         "query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
         "includeAll": true,
+        "allValue": ".*",
         "multi": true,
         "current": {
           "text": "All",
@@ -289,6 +295,7 @@
     "from": "now-6h",
     "to": "now"
   },
+  "refresh": "30s",
   "timepicker": {},
   "timezone": "utc",
   "title": "toad-eye / Provider Health",