toad-eye 2.4.5 → 2.6.0

package/templates/grafana/dashboards/model-comparison.json

@@ -1,5 +1,7 @@
 {
-  "annotations": { "list": [] },
+  "annotations": {
+    "list": []
+  },
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
@@ -9,11 +11,19 @@
     {
       "title": "Latency by Model (p95)",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le, gen_ai_request_model))",
+          "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_milliseconds_bucket{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (le, gen_ai_request_model))",
           "legendFormat": "p95 {{ gen_ai_request_model }}",
           "refId": "A"
         }
@@ -21,7 +31,11 @@
       "fieldConfig": {
         "defaults": {
           "unit": "ms",
-          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineWidth": 2
+          }
         },
         "overrides": []
       }
@@ -29,11 +43,19 @@
     {
       "title": "Cost by Model (avg per request)",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_request_cost_USD_sum{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_request_cost_USD_count{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model)",
+          "expr": "sum(rate(gen_ai_client_request_cost_USD_sum{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_request_cost_USD_count{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model)",
           "legendFormat": "{{ gen_ai_request_model }}",
           "refId": "A"
         }
@@ -41,7 +63,11 @@
       "fieldConfig": {
         "defaults": {
           "unit": "currencyUSD",
-          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineWidth": 2
+          }
         },
         "overrides": []
       }
@@ -49,11 +75,19 @@
     {
       "title": "Error Rate by Model (%)",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) * 100",
+          "expr": "sum(rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) * 100",
           "legendFormat": "{{ gen_ai_request_model }}",
           "refId": "A"
         }
@@ -61,7 +95,11 @@
       "fieldConfig": {
         "defaults": {
           "unit": "percent",
-          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineWidth": 2
+          }
         },
         "overrides": []
       }
@@ -69,11 +107,19 @@
     {
       "title": "Throughput by Model (req/min)",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_request_model) * 60",
+          "expr": "sum(rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[1m])) by (gen_ai_request_model) * 60",
           "legendFormat": "{{ gen_ai_request_model }}",
           "refId": "A"
         }
@@ -81,7 +127,11 @@
       "fieldConfig": {
         "defaults": {
           "unit": "reqpm",
-          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineWidth": 2
+          }
         },
         "overrides": []
       }
@@ -89,11 +139,19 @@
     {
       "title": "Tokens per Request by Model",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum(rate(gen_ai_client_token_usage_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model)",
+          "expr": "sum(rate(gen_ai_client_token_usage_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model) / sum(rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\", gen_ai_request_model=~\"$model\"}[5m])) by (gen_ai_request_model)",
           "legendFormat": "{{ gen_ai_request_model }}",
           "refId": "A"
         }
@@ -101,7 +159,11 @@
       "fieldConfig": {
         "defaults": {
           "unit": "short",
-          "custom": { "drawStyle": "line", "fillOpacity": 15, "lineWidth": 2 }
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "lineWidth": 2
+          }
         },
         "overrides": []
       }
@@ -112,29 +174,60 @@
   "tags": ["llm", "observability", "toad-eye"],
   "templating": {
     "list": [
+      {
+        "name": "service",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": "label_values(gen_ai_client_requests_total, job)",
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true
+      },
       {
         "name": "provider",
         "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
         "query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
         "includeAll": true,
         "allValue": ".*",
-        "current": { "text": "All", "value": "$__all" },
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
         "multi": true
       },
       {
         "name": "model",
         "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
         "query": "label_values(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}, gen_ai_request_model)",
         "includeAll": true,
         "allValue": ".*",
-        "current": { "text": "All", "value": "$__all" },
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
         "multi": true
       }
     ]
   },
-  "time": { "from": "now-15m", "to": "now" },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
   "timepicker": {},
   "timezone": "browser",
   "title": "toad-eye: Model Comparison",

package/templates/grafana/dashboards/provider-health.json

@@ -1,5 +1,7 @@
 {
-  "annotations": { "list": [] },
+  "annotations": {
+    "list": []
+  },
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 1,
@@ -10,11 +12,19 @@
       "title": "Provider Status",
       "description": "healthy (< 10% errors) / degraded (10-50%) / down (> 50%)",
       "type": "stat",
-      "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
           "legendFormat": "{{gen_ai_provider_name}}",
           "refId": "A"
         }
@@ -25,9 +35,18 @@
           "thresholds": {
             "mode": "absolute",
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 0.1 },
-              { "color": "red", "value": 0.5 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 0.5
+              }
             ]
           },
           "mappings": [
@@ -36,7 +55,10 @@
               "options": {
                 "from": 0,
                 "to": 0.1,
-                "result": { "text": "HEALTHY", "color": "green" }
+                "result": {
+                  "text": "HEALTHY",
+                  "color": "green"
+                }
               }
             },
             {
@@ -44,7 +66,10 @@
               "options": {
                 "from": 0.1,
                 "to": 0.5,
-                "result": { "text": "DEGRADED", "color": "yellow" }
+                "result": {
+                  "text": "DEGRADED",
+                  "color": "yellow"
+                }
               }
             },
             {
@@ -52,7 +77,10 @@
               "options": {
                 "from": 0.5,
                 "to": 1,
-                "result": { "text": "DOWN", "color": "red" }
+                "result": {
+                  "text": "DOWN",
+                  "color": "red"
+                }
               }
             }
           ]
@@ -63,11 +91,19 @@
     {
       "title": "Request Volume by Provider",
       "type": "timeseries",
-      "gridPos": { "h": 6, "w": 16, "x": 8, "y": 0 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 6,
+        "w": 16,
+        "x": 8,
+        "y": 0
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
           "legendFormat": "{{gen_ai_provider_name}}",
           "refId": "A"
         }
@@ -75,7 +111,9 @@
       "fieldConfig": {
         "defaults": {
           "unit": "reqps",
-          "custom": { "fillOpacity": 15 }
+          "custom": {
+            "fillOpacity": 15
+          }
         },
         "overrides": []
       }
@@ -83,11 +121,19 @@
     {
       "title": "Error Rate by Provider (5m window)",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "expr": "sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
           "legendFormat": "{{gen_ai_provider_name}} error rate",
           "refId": "A"
         }
@@ -95,13 +141,24 @@
       "fieldConfig": {
         "defaults": {
           "unit": "percentunit",
-          "custom": { "fillOpacity": 10 },
+          "custom": {
+            "fillOpacity": 10
+          },
           "thresholds": {
             "mode": "absolute",
             "steps": [
-              { "color": "green", "value": null },
-              { "color": "yellow", "value": 0.1 },
-              { "color": "red", "value": 0.5 }
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 0.5
+              }
             ]
           }
         },
@@ -111,11 +168,19 @@
     {
       "title": "Error Breakdown by Type",
       "type": "timeseries",
-      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
-          "expr": "sum by (gen_ai_provider_name, error_type) (rate(gen_ai_client_errors_total{gen_ai_provider_name=~\"$provider\"}[5m]))",
+          "expr": "sum by (gen_ai_provider_name, error_type) (rate(gen_ai_client_errors_total{job=~\"$service\", gen_ai_provider_name=~\"$provider\"}[5m]))",
           "legendFormat": "{{gen_ai_provider_name}} — {{error_type}}",
           "refId": "A"
         }
@@ -123,7 +188,12 @@
       "fieldConfig": {
         "defaults": {
           "unit": "reqps",
-          "custom": { "fillOpacity": 20, "stacking": { "mode": "normal" } }
+          "custom": {
+            "fillOpacity": 20,
+            "stacking": {
+              "mode": "normal"
+            }
+          }
         },
         "overrides": []
       }
@@ -132,8 +202,16 @@
       "title": "Provider Availability (24h / 7d / 30d)",
       "description": "Percentage of time with error rate below 10%",
       "type": "table",
-      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 },
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 14
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
       "targets": [
         {
           "expr": "1 - (sum by (gen_ai_provider_name) (rate(gen_ai_client_errors_total[24h])) / sum by (gen_ai_provider_name) (rate(gen_ai_client_requests_total[24h])))",
@@ -161,7 +239,10 @@
         }
       ],
       "fieldConfig": {
-        "defaults": { "unit": "percentunit", "decimals": 2 },
+        "defaults": {
+          "unit": "percentunit",
+          "decimals": 2
+        },
         "overrides": []
       }
     }
@@ -170,19 +251,44 @@
   "tags": ["toad-eye", "provider-health"],
   "templating": {
     "list": [
+      {
+        "name": "service",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "query": "label_values(gen_ai_client_requests_total, job)",
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "multi": true
+      },
       {
         "name": "provider",
         "type": "query",
-        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
         "query": "label_values(gen_ai_client_requests_total, gen_ai_provider_name)",
         "includeAll": true,
         "multi": true,
-        "current": { "text": "All", "value": "$__all" },
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
         "regex": ""
       }
     ]
   },
-  "time": { "from": "now-6h", "to": "now" },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
   "timepicker": {},
   "timezone": "utc",
   "title": "toad-eye / Provider Health",