sixtytwo-cli 0.3.6__cp314-cp314-win_amd64.whl

sixtytwo/__main__.py

@@ -0,0 +1,5 @@
+from .cli import unified_main as main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sixtytwo/grafana/sixtytwo-gpu-telemetry.json

@@ -0,0 +1,579 @@
+{
+  "annotations": {"list": []},
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "schemaVersion": 39,
+  "tags": ["sixtytwo", "gpu", "telemetry", "hardware"],
+  "time": {"from": "now-1h", "to": "now"},
+  "timepicker": {},
+  "timezone": "",
+  "title": "Sixtytwo // GPU Hardware Telemetry",
+  "uid": "sixtytwo-gpu-telemetry",
+  "version": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "datasource",
+        "type": "datasource",
+        "query": "prometheus",
+        "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
+        "hide": 0
+      },
+      {
+        "name": "node",
+        "type": "query",
+        "label": "Node",
+        "datasource": {"type": "prometheus", "uid": "${datasource}"},
+        "query": "label_values(sixtytwo_gpu_up, node_id)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {"selected": true, "text": "All", "value": "$__all"}
+      },
+      {
+        "name": "vendor",
+        "type": "query",
+        "label": "Vendor",
+        "datasource": {"type": "prometheus", "uid": "${datasource}"},
+        "query": "label_values(sixtytwo_gpu_up{node_id=~\"$node\"}, vendor)",
+        "refresh": 2,
+        "includeAll": true,
+        "multi": true,
+        "current": {"selected": true, "text": "All", "value": "$__all"}
+      }
+    ]
+  },
+  "panels": [
+    {
+      "type": "row",
+      "title": "Fleet Overview",
+      "id": 100,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 0},
+      "collapsed": false
+    },
+    {
+      "type": "stat",
+      "title": "Total GPUs Online",
+      "id": 1,
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {"expr": "count(sixtytwo_gpu_up{node_id=~\"$node\",vendor=~\"$vendor\"} == 1)", "refId": "A", "legendFormat": ""}
+      ],
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "none"},
+      "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}}}
+    },
+    {
+      "type": "stat",
+      "title": "GPUs Unreachable",
+      "id": 2,
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {"expr": "count(sixtytwo_gpu_up{node_id=~\"$node\",vendor=~\"$vendor\"} == 0) or vector(0)", "refId": "A", "legendFormat": ""}
+      ],
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "none"},
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "red", "value": 1}
+          ]}
+        }
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Avg GPU Utilization",
+      "id": 3,
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {"expr": "avg(sixtytwo_gpu_utilization_ratio{node_id=~\"$node\",vendor=~\"$vendor\"})", "refId": "A"}
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0, "max": 1,
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "red", "value": null},
+            {"color": "yellow", "value": 0.5},
+            {"color": "green", "value": 0.8}
+          ]}
+        }
+      },
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "area"}
+    },
+    {
+      "type": "stat",
+      "title": "Nodes w/ ECC Uncorrected Errors",
+      "id": 4,
+      "gridPos": {"h": 4, "w": 4, "x": 12, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "count(increase(sixtytwo_gpu_ecc_errors_uncorrected_volatile_total{node_id=~\"$node\"}[10m]) > 0) or vector(0)",
+          "refId": "A",
+          "legendFormat": ""
+        }
+      ],
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "none"},
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "red", "value": 1}
+          ]}
+        }
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Max GPU Temperature",
+      "id": 5,
+      "gridPos": {"h": 4, "w": 4, "x": 16, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {"expr": "max(sixtytwo_gpu_temperature_celsius{node_id=~\"$node\",vendor=~\"$vendor\"})", "refId": "A"}
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "yellow", "value": 75},
+            {"color": "red", "value": 85}
+          ]}
+        }
+      },
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "area"}
+    },
+    {
+      "type": "stat",
+      "title": "Total Fleet Power Draw",
+      "id": 6,
+      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 1},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {"expr": "sum(sixtytwo_gpu_power_watts{node_id=~\"$node\",vendor=~\"$vendor\"})", "refId": "A"}
+      ],
+      "fieldConfig": {"defaults": {"unit": "watt"}},
+      "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "graphMode": "area"}
+    },
+    {
+      "type": "row",
+      "title": "Utilization",
+      "id": 200,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 5},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Compute Utilization",
+      "id": 10,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_utilization_ratio{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}} ({{name}})"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0, "max": 1,
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Memory Utilization",
+      "id": 11,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_memory_utilization_ratio{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0, "max": 1,
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "row",
+      "title": "Memory",
+      "id": 300,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 14},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Memory Used",
+      "id": 20,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_memory_used_bytes{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Memory Used (% of total)",
+      "id": 21,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_memory_used_bytes{node_id=~\"$node\",vendor=~\"$vendor\"} / sixtytwo_gpu_memory_total_bytes{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0, "max": 1,
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "yellow", "value": 0.85},
+            {"color": "red", "value": 0.95}
+          ]},
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "row",
+      "title": "Thermals & Power",
+      "id": 400,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 23},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Temperature",
+      "id": 30,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 24},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_temperature_celsius{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius",
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "yellow", "value": 75},
+            {"color": "red", "value": 85}
+          ]},
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "thresholdsStyle": {"mode": "line"}}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU Power Draw vs Limit",
+      "id": 31,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 24},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_power_watts{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}} draw"
+        },
+        {
+          "expr": "sixtytwo_gpu_power_limit_watts{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "B",
+          "legendFormat": "{{node_id}} GPU{{index}} limit"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "watt",
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 5}
+        },
+        "overrides": [
+          {
+            "matcher": {"id": "byFrameRefID", "options": "B"},
+            "properties": [
+              {"id": "custom.lineStyle", "value": {"dash": [10, 10], "fill": "dash"}},
+              {"id": "custom.fillOpacity", "value": 0}
+            ]
+          }
+        ]
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "SM & Memory Clock Frequencies",
+      "id": 32,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_sm_clock_mhz{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}} SM"
+        },
+        {
+          "expr": "sixtytwo_gpu_mem_clock_mhz{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "B",
+          "legendFormat": "{{node_id}} GPU{{index}} mem"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "megahertz",
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 5}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Fan Speed",
+      "id": 33,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_fan_speed_ratio{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0, "max": 1,
+          "custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 5}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "row",
+      "title": "ECC Errors",
+      "id": 500,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 40},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "Uncorrectable ECC Errors (volatile, rate)",
+      "id": 40,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 41},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "increase(sixtytwo_gpu_ecc_errors_uncorrected_volatile_total{node_id=~\"$node\",vendor=~\"$vendor\"}[5m])",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}} DBE"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "green", "value": null},
+            {"color": "red", "value": 1}
+          ]},
+          "custom": {
+            "drawStyle": "bars",
+            "lineWidth": 1,
+            "fillOpacity": 80,
+            "thresholdsStyle": {"mode": "area"}
+          }
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Correctable ECC Errors (volatile, rate)",
+      "id": 41,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 41},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "increase(sixtytwo_gpu_ecc_errors_corrected_volatile_total{node_id=~\"$node\",vendor=~\"$vendor\"}[5m])",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}} SBE"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {"drawStyle": "bars", "lineWidth": 1, "fillOpacity": 60}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "table",
+      "title": "Aggregate ECC Error Totals (lifetime)",
+      "id": 42,
+      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 49},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_ecc_errors_corrected_aggregate_total{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "instant": true,
+          "format": "table",
+          "legendFormat": "corrected"
+        },
+        {
+          "expr": "sixtytwo_gpu_ecc_errors_uncorrected_aggregate_total{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "B",
+          "instant": true,
+          "format": "table",
+          "legendFormat": "uncorrected"
+        }
+      ],
+      "transformations": [
+        {"id": "merge", "options": {}},
+        {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "job": true, "instance": true}}}
+      ]
+    },
+    {
+      "type": "row",
+      "title": "PCIe",
+      "id": 600,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 57},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "PCIe Link Generation",
+      "id": 50,
+      "gridPos": {"h": 6, "w": 12, "x": 0, "y": 58},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_pcie_link_gen{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "red", "value": null},
+            {"color": "yellow", "value": 4},
+            {"color": "green", "value": 5}
+          ]},
+          "custom": {"drawStyle": "line", "lineWidth": 1, "thresholdsStyle": {"mode": "line"}}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "PCIe Link Width (lanes)",
+      "id": 51,
+      "gridPos": {"h": 6, "w": 12, "x": 12, "y": 58},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sixtytwo_gpu_pcie_link_width{node_id=~\"$node\",vendor=~\"$vendor\"}",
+          "refId": "A",
+          "legendFormat": "{{node_id}} GPU{{index}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "min": 0,
+          "thresholds": {"mode": "absolute", "steps": [
+            {"color": "red", "value": null},
+            {"color": "yellow", "value": 8},
+            {"color": "green", "value": 16}
+          ]},
+          "custom": {"drawStyle": "line", "lineWidth": 1, "thresholdsStyle": {"mode": "line"}}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "row",
+      "title": "Runtime Monitoring Events",
+      "id": 700,
+      "gridPos": {"h": 1, "w": 24, "x": 0, "y": 64},
+      "collapsed": false
+    },
+    {
+      "type": "timeseries",
+      "title": "Monitoring Event Rate by Type",
+      "id": 60,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 65},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sum by (event_type) (increase(sixtytwo_monitoring_events_total{node_id=~\"$node\"}[30m]))",
+          "refId": "A",
+          "legendFormat": "{{event_type}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {"drawStyle": "bars", "lineWidth": 1, "fillOpacity": 70, "stacking": {"mode": "value"}}
+        }
+      },
+      "options": {"legend": {"displayMode": "table", "placement": "bottom"}}
+    },
+    {
+      "type": "table",
+      "title": "Monitoring Event Counts by Node",
+      "id": 61,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 65},
+      "datasource": {"type": "prometheus", "uid": "${datasource}"},
+      "targets": [
+        {
+          "expr": "sum by (node_id, event_type, severity) (sixtytwo_monitoring_events_total{node_id=~\"$node\"})",
+          "refId": "A",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "organize", "options": {"excludeByName": {"Time": true, "__name__": true, "job": true, "instance": true}}}
+      ]
+    }
+  ]
+}