stado 0.4.364__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. stado/__init__.py +1 -0
  2. stado/_catalog/__init__.py +0 -0
  3. stado/_catalog/gpu_sku.py +256 -0
  4. stado/cli.py +1321 -0
  5. stado/cloud_function/__init__.py +0 -0
  6. stado/cloud_function/main.py +111 -0
  7. stado/config.py +291 -0
  8. stado/coordinator.py +226 -0
  9. stado/coverage/__init__.py +290 -0
  10. stado/coverage/cli.py +123 -0
  11. stado/coverage/failures.py +167 -0
  12. stado/dashboard.py +236 -0
  13. stado/dashboard_summary/__init__.py +169 -0
  14. stado/dashboard_summary/status_view.py +86 -0
  15. stado/deploy/__init__.py +0 -0
  16. stado/deploy/bootstrap.py +198 -0
  17. stado/deploy/host_health_beacon.sh +154 -0
  18. stado/deploy/local_install.py +250 -0
  19. stado/deploy/pre_start_cleanup.sh +79 -0
  20. stado/deploy/templates/wisent-agent.service.tmpl +31 -0
  21. stado/deploy/templates/wisent-host-health.service.tmpl +14 -0
  22. stado/deploy/templates/wisent-host-health.timer.tmpl +11 -0
  23. stado/deploy/templates/wisent-upgrade.service.tmpl +22 -0
  24. stado/deploy/templates/wisent-upgrade.timer.tmpl +12 -0
  25. stado/deploy/watchdog/__init__.py +1 -0
  26. stado/deploy/watchdog/cli.py +140 -0
  27. stado/failure_fixer/__init__.py +224 -0
  28. stado/failure_fixer/cli.py +112 -0
  29. stado/mcp/__init__.py +1 -0
  30. stado/mcp/server.py +238 -0
  31. stado/models.py +224 -0
  32. stado/monitor/__init__.py +4 -0
  33. stado/monitor/alerts.py +72 -0
  34. stado/monitor/billing.py +246 -0
  35. stado/monitor/heartbeat_guard.py +299 -0
  36. stado/monitor/monitor.py +300 -0
  37. stado/monitor/reap/__init__.py +0 -0
  38. stado/monitor/reap/helpers.py +220 -0
  39. stado/monitor/reap/run_reaper.py +68 -0
  40. stado/profiles/__init__.py +157 -0
  41. stado/profiles/ai_toolkit_zimage.json +12 -0
  42. stado/providers/__init__.py +14 -0
  43. stado/providers/aws.py +101 -0
  44. stado/providers/azure.py +251 -0
  45. stado/providers/azure_helpers/__init__.py +0 -0
  46. stado/providers/azure_helpers/network.py +58 -0
  47. stado/providers/base.py +41 -0
  48. stado/providers/gcp/__init__.py +277 -0
  49. stado/providers/gcp/stockout.py +186 -0
  50. stado/providers/local/__init__.py +0 -0
  51. stado/providers/local/disk/__init__.py +4 -0
  52. stado/providers/local/disk/gate.py +287 -0
  53. stado/providers/local/disk/staging.py +181 -0
  54. stado/providers/local/gcp_self.py +57 -0
  55. stado/providers/local/helpers/__init__.py +300 -0
  56. stado/providers/local/helpers/gpu_probe.py +104 -0
  57. stado/providers/local/hf_rate.py +115 -0
  58. stado/providers/local/slots.py +556 -0
  59. stado/providers/local/version_check.py +219 -0
  60. stado/providers/local_agent.py +452 -0
  61. stado/providers/vast/__init__.py +299 -0
  62. stado/providers/vast/_auth.py +40 -0
  63. stado/queue/__init__.py +2 -0
  64. stado/queue/azure_blob.py +159 -0
  65. stado/queue/capacity.py +185 -0
  66. stado/queue/listing/__init__.py +178 -0
  67. stado/queue/migrations.py +87 -0
  68. stado/queue/runs/__init__.py +122 -0
  69. stado/queue/storage.py +313 -0
  70. stado/queue/submit.py +327 -0
  71. stado/queue/tracking/__init__.py +4 -0
  72. stado/queue/tracking/tombstone.py +48 -0
  73. stado/scheduler/__init__.py +1 -0
  74. stado/scheduler/cost.py +217 -0
  75. stado/scheduler/dispatch/__init__.py +0 -0
  76. stado/scheduler/dispatch/agent.py +236 -0
  77. stado/scheduler/dispatch/quota_replies.py +298 -0
  78. stado/scheduler/dispatch/quota_request.py +267 -0
  79. stado/scheduler/dispatch/quota_skus.py +295 -0
  80. stado/scheduler/makespan/__init__.py +282 -0
  81. stado/scheduler/makespan/_history.py +102 -0
  82. stado/scheduler/quota.py +218 -0
  83. stado/scheduler/scheduler.py +333 -0
  84. stado/scheduler/skip_done.py +95 -0
  85. stado/schedules/__init__.py +16 -0
  86. stado/schedules/fire.py +104 -0
  87. stado/schedules/model.py +133 -0
  88. stado/schedules/store.py +117 -0
  89. stado/sizing/__init__.py +346 -0
  90. stado/targets/__init__.py +216 -0
  91. stado/targets/registry.example.json +61 -0
  92. stado/templates/__init__.py +0 -0
  93. stado/templates/startup_cpu.sh +44 -0
  94. stado/templates/startup_gpu.sh +93 -0
  95. stado/templates/startup_gpu_agent.sh +191 -0
  96. stado/templates/startup_gpu_agent_azure.sh +67 -0
  97. stado-0.4.364.dist-info/METADATA +25 -0
  98. stado-0.4.364.dist-info/RECORD +102 -0
  99. stado-0.4.364.dist-info/WHEEL +5 -0
  100. stado-0.4.364.dist-info/entry_points.txt +6 -0
  101. stado-0.4.364.dist-info/licenses/LICENSE +202 -0
  102. stado-0.4.364.dist-info/top_level.txt +1 -0
stado/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.3.0"
stado/_catalog/__init__.py ADDED
File without changes
stado/_catalog/gpu_sku.py ADDED
@@ -0,0 +1,256 @@
1
+ """GPU SKU catalog for Azure and GCE. Imported by models.py.
2
+
3
+ Covers every public NVIDIA / AMD GPU VM family as of 2026-05 across both
4
+ clouds: K80, P100, V100, T4, A10, A100-40, A100-80, H100, H200, L4, B200,
5
+ GB200, MI300X. Keeps models.py under the 300-line cap by holding the
6
+ multi-generation Azure NC/ND/NV SKU tables here.
7
+ """
8
+ from __future__ import annotations
9
+
10
+
11
+ GPU_SIZING = {
12
+ "gcp": {
13
+ 12: ("n1-standard-4", "nvidia-tesla-k80"),
14
+ 16: ("n1-standard-4", "nvidia-tesla-t4"),
15
+ 24: ("g2-standard-4", "nvidia-l4"),
16
+ 32: ("n1-standard-8", "nvidia-tesla-v100"),
17
+ 40: ("a2-highgpu-1g", "nvidia-tesla-a100"),
18
+ 80: ("a2-ultragpu-1g", "nvidia-a100-80gb"),
19
+ 94: ("a3-highgpu-1g", "nvidia-h100-80gb"),
20
+ 141: ("a3-ultragpu-8g", "nvidia-h200-141gb"),
21
+ 180: ("a4-highgpu-8g", "nvidia-b200-180gb"),
22
+ 192: ("a4x-highgpu-4g", "nvidia-gb200-192gb"),
23
+ },
24
+ "azure": {
25
+ 12: ("Standard_NC6", "nvidia-tesla-k80"),
26
+ 16: ("Standard_NC6s_v3", "nvidia-tesla-v100"),
27
+ 22: ("Standard_NC4as_T4_v3", "nvidia-tesla-t4"),
28
+ 24: ("Standard_NC8ads_A10_v4", "nvidia-a10"),
29
+ 40: ("Standard_NC24ads_A100_v4", "nvidia-a100-80gb"),
30
+ 80: ("Standard_NC24ads_A100_v4", "nvidia-a100-80gb"),
31
+ 94: ("Standard_NC40ads_H100_v5", "nvidia-h100-94gb"),
32
+ 141: ("Standard_ND96isr_H200_v5", "nvidia-h200-141gb"),
33
+ 180: ("Standard_ND96isr_B200_v6", "nvidia-b200-180gb"),
34
+ 192: ("Standard_ND96isr_MI300X_v5", "amd-mi300x-192gb"),
35
+ },
36
+ }
37
+
38
+
39
+ AZURE_VM_TO_ACCEL = {
40
+ "Standard_NC6": ("nvidia-tesla-k80", 1),
41
+ "Standard_NC12": ("nvidia-tesla-k80", 2),
42
+ "Standard_NC24": ("nvidia-tesla-k80", 4),
43
+ "Standard_NC24r": ("nvidia-tesla-k80", 4),
44
+ "Standard_NC6s_v2": ("nvidia-tesla-p100", 1),
45
+ "Standard_NC12s_v2": ("nvidia-tesla-p100", 2),
46
+ "Standard_NC24s_v2": ("nvidia-tesla-p100", 4),
47
+ "Standard_NC24rs_v2": ("nvidia-tesla-p100", 4),
48
+ "Standard_NC6s_v3": ("nvidia-tesla-v100", 1),
49
+ "Standard_NC12s_v3": ("nvidia-tesla-v100", 2),
50
+ "Standard_NC24s_v3": ("nvidia-tesla-v100", 4),
51
+ "Standard_NC24rs_v3": ("nvidia-tesla-v100", 4),
52
+ "Standard_NC4as_T4_v3": ("nvidia-tesla-t4", 1),
53
+ "Standard_NC8as_T4_v3": ("nvidia-tesla-t4", 1),
54
+ "Standard_NC16as_T4_v3": ("nvidia-tesla-t4", 1),
55
+ "Standard_NC64as_T4_v3": ("nvidia-tesla-t4", 4),
56
+ "Standard_NC8ads_A10_v4": ("nvidia-a10", 1),
57
+ "Standard_NC16ads_A10_v4": ("nvidia-a10", 1),
58
+ "Standard_NC32ads_A10_v4": ("nvidia-a10", 1),
59
+ "Standard_NC24ads_A100_v4": ("nvidia-a100-80gb", 1),
60
+ "Standard_NC48ads_A100_v4": ("nvidia-a100-80gb", 2),
61
+ "Standard_NC96ads_A100_v4": ("nvidia-a100-80gb", 4),
62
+ "Standard_NC40ads_H100_v5": ("nvidia-h100-94gb", 1),
63
+ "Standard_NC80adis_H100_v5": ("nvidia-h100-94gb", 2),
64
+ "Standard_NCC40ads_H100_v5": ("nvidia-h100-94gb", 1),
65
+ "Standard_ND40rs_v2": ("nvidia-tesla-v100-32gb", 8),
66
+ "Standard_ND96asr_v4": ("nvidia-tesla-a100", 8),
67
+ "Standard_ND96amsr_A100_v4": ("nvidia-a100-80gb", 8),
68
+ "Standard_ND96is_H100_v5": ("nvidia-h100-80gb", 8),
69
+ "Standard_ND96isr_H100_v5": ("nvidia-h100-80gb", 8),
70
+ "Standard_ND96isr_H200_v5": ("nvidia-h200-141gb", 8),
71
+ "Standard_ND96isr_MI300X_v5": ("amd-mi300x-192gb", 8),
72
+ "Standard_ND96isr_B200_v6": ("nvidia-b200-180gb", 8),
73
+ "Standard_ND96isr_GB200_v6": ("nvidia-gb200-192gb", 8),
74
+ "Standard_ND72isr_GB200_v6": ("nvidia-gb200-192gb", 8),
75
+ "Standard_NV6ads_A10_v5": ("nvidia-a10", 1),
76
+ "Standard_NV12ads_A10_v5": ("nvidia-a10", 1),
77
+ "Standard_NV18ads_A10_v5": ("nvidia-a10", 1),
78
+ "Standard_NV36ads_A10_v5": ("nvidia-a10", 1),
79
+ "Standard_NV36adms_A10_v5": ("nvidia-a10", 1),
80
+ "Standard_NV72ads_A10_v5": ("nvidia-a10", 2),
81
+ "Standard_NV4ads_V710_v5": ("amd-radeonpro-v710", 1),
82
+ "Standard_NV8ads_V710_v5": ("amd-radeonpro-v710", 1),
83
+ "Standard_NV12ads_V710_v5": ("amd-radeonpro-v710", 1),
84
+ "Standard_NV24ads_V710_v5": ("amd-radeonpro-v710", 1),
85
+ "Standard_NV28adms_V710_v5": ("amd-radeonpro-v710", 1),
86
+ }
87
+
88
+
89
+ AZURE_QUOTA_FAMILY_TO_ACCEL = {
90
+ "standardNCFamily": "nvidia-tesla-k80",
91
+ "standardNCSv2Family": "nvidia-tesla-p100",
92
+ "standardNCSv3Family": "nvidia-tesla-v100",
93
+ "standardNCPromoFamily": "nvidia-tesla-k80",
94
+ "Standard NCASv3_T4 Family": "nvidia-tesla-t4",
95
+ "standardNCASv3Family": "nvidia-tesla-t4",
96
+ "standardNCASv4Family": "nvidia-tesla-t4",
97
+ "StandardNCADSA10v4Family": "nvidia-a10",
98
+ "StandardNCADSA100v4Family": "nvidia-a100-80gb",
99
+ "StandardNCadsH100v5Family": "nvidia-h100-94gb",
100
+ "StandardNCCads2023Family": "nvidia-h100-94gb",
101
+ "standardNDSFamily": "nvidia-tesla-p40",
102
+ "standardNDSv2Family": "nvidia-tesla-v100-32gb",
103
+ "standardNDSv3Family": "nvidia-tesla-v100",
104
+ "standard NDAMSv4_A100Family": "nvidia-a100-80gb",
105
+ "Standard NDASv4_A100 Family": "nvidia-tesla-a100",
106
+ "standardNDSH100v5Family": "nvidia-h100-80gb",
107
+ "standardNDISRH200V5Family": "nvidia-h200-141gb",
108
+ "standardNDISRGB200V6NDRFamily": "nvidia-b200-180gb",
109
+ "standardNDISRGB300V6Family": "nvidia-gb200-192gb",
110
+ "standardNDISRGB300G5V6Family": "nvidia-gb200-192gb",
111
+ "standardNDISv5MI300XFamily": "amd-mi300x-192gb",
112
+ "standardNVFamily": "nvidia-tesla-m60",
113
+ "standardNVSv2Family": "nvidia-tesla-m60",
114
+ "standardNVSv3Family": "nvidia-tesla-m60",
115
+ "standardNVSv4Family": "amd-radeonpro-v520",
116
+ "standardNVPromoFamily": "nvidia-tesla-m60",
117
+ "StandardNVADSA10v5Family": "nvidia-a10",
118
+ "StandardNVadsV710v5Family": "amd-radeonpro-v710",
119
+ }
120
+
121
+
122
+ GPU_HOURLY_RATE_USD = {
123
+ "nvidia-tesla-k80": 0.45,
124
+ "nvidia-tesla-p100": 1.46,
125
+ "nvidia-tesla-p40": 1.30,
126
+ "nvidia-tesla-v100": 2.48,
127
+ "nvidia-tesla-v100-32gb": 3.06,
128
+ "nvidia-tesla-t4": 0.35,
129
+ "nvidia-l4": 0.71,
130
+ "nvidia-a10": 1.20,
131
+ "nvidia-tesla-a100": 2.93,
132
+ "nvidia-a100-80gb": 3.67,
133
+ "nvidia-h100-80gb": 11.06,
134
+ "nvidia-h100-94gb": 11.06,
135
+ "nvidia-h200-141gb": 13.50,
136
+ "nvidia-b200-180gb": 22.00,
137
+ "nvidia-gb200-192gb": 28.00,
138
+ "amd-mi300x-192gb": 9.00,
139
+ "amd-radeonpro-v520": 0.50,
140
+ "amd-radeonpro-v710": 0.70,
141
+ "nvidia-tesla-m60": 1.20,
142
+ # Owned RTX PRO 6000 Blackwell Workstation Edition (96 GB GDDR7,
143
+ # 600 W TGP). Hardware is sunk cost; the hourly rate models only
144
+ # marginal electricity at California commercial rates: 0.6 kW x
145
+ # $0.30/kWh = $0.18/hr at full GPU power. Used by scheduler/cost.py
146
+ # for per-job cost accounting on this box.
147
+ "nvidia-rtx-pro-6000": 0.18,
148
+ }
149
+
150
+
151
+ SPOT_DISCOUNT = {
152
+ "nvidia-tesla-k80": 0.30,
153
+ "nvidia-tesla-p100": 0.30,
154
+ "nvidia-tesla-p40": 0.30,
155
+ "nvidia-tesla-v100": 0.30,
156
+ "nvidia-tesla-v100-32gb": 0.30,
157
+ "nvidia-tesla-t4": 0.49,
158
+ "nvidia-l4": 0.40,
159
+ "nvidia-a10": 0.40,
160
+ "nvidia-tesla-a100": 0.49,
161
+ "nvidia-a100-80gb": 0.54,
162
+ "nvidia-h100-80gb": 0.45,
163
+ "nvidia-h100-94gb": 0.45,
164
+ "nvidia-h200-141gb": 0.50,
165
+ "nvidia-b200-180gb": 0.55,
166
+ "nvidia-gb200-192gb": 0.60,
167
+ "amd-mi300x-192gb": 0.50,
168
+ "amd-radeonpro-v520": 0.30,
169
+ "amd-radeonpro-v710": 0.30,
170
+ "nvidia-tesla-m60": 0.30,
171
+ # Owned hardware — no spot tier; electricity costs the same regardless.
172
+ "nvidia-rtx-pro-6000": 1.0,
173
+ }
174
+
175
+
176
+ VM_BUNDLE_HOURLY_RATE_USD = {
177
+ "a2-highgpu-1g": (1.50, 0.37),
178
+ "a2-ultragpu-1g": (1.85, 0.55),
179
+ "a2-highgpu-2g": (3.00, 0.74),
180
+ "a2-ultragpu-2g": (3.70, 1.10),
181
+ "a2-highgpu-4g": (6.00, 1.48),
182
+ "a2-ultragpu-4g": (7.40, 2.20),
183
+ "a2-highgpu-8g": (12.00, 2.96),
184
+ "a2-ultragpu-8g": (14.80, 4.40),
185
+ "a3-highgpu-1g": (3.00, 1.20),
186
+ "a3-highgpu-2g": (6.00, 2.40),
187
+ "a3-highgpu-4g": (12.00, 4.80),
188
+ "a3-highgpu-8g": (8.00, 3.20),
189
+ "a3-megagpu-8g": (10.00, 4.00),
190
+ "a3-edgegpu-8g": (9.00, 3.60),
191
+ "a3-ultragpu-8g": (12.00, 4.80),
192
+ "a4-highgpu-8g": (20.00, 8.00),
193
+ "a4x-highgpu-4g": (24.00, 9.60),
194
+ "n1-standard-4": (0.20, 0.06),
195
+ "n1-standard-8": (0.40, 0.12),
196
+ "g2-standard-4": (0.30, 0.12),
197
+ "g2-standard-8": (0.60, 0.24),
198
+ }
199
+
200
+
201
+ GPU_TYPE_TO_MACHINE_TYPE = {
202
+ "nvidia-tesla-k80": "n1-standard-4",
203
+ "nvidia-tesla-p100": "n1-standard-8",
204
+ "nvidia-tesla-p40": "n1-standard-8",
205
+ "nvidia-tesla-v100": "n1-standard-8",
206
+ "nvidia-tesla-v100-32gb": "n1-standard-8",
207
+ "nvidia-tesla-t4": "n1-standard-4",
208
+ "nvidia-l4": "g2-standard-4",
209
+ "nvidia-tesla-a100": "a2-highgpu-1g",
210
+ "nvidia-a100-80gb": "a2-ultragpu-1g",
211
+ "nvidia-a10": "n1-standard-8",
212
+ "nvidia-h100-80gb": "a3-highgpu-8g",
213
+ "nvidia-h100-94gb": "a3-highgpu-1g",
214
+ "nvidia-h200-141gb": "a3-ultragpu-8g",
215
+ "nvidia-b200-180gb": "a4-highgpu-8g",
216
+ "nvidia-gb200-192gb": "a4x-highgpu-4g",
217
+ "amd-mi300x-192gb": "a4-highgpu-8g",
218
+ }
219
+
220
+
221
+ AZURE_VM_HOURLY_RATE_USD = {
222
+ "Standard_NC4as_T4_v3": (0.526, 0.105),
223
+ "Standard_NC8as_T4_v3": (0.752, 0.150),
224
+ "Standard_NC16as_T4_v3": (1.204, 0.241),
225
+ "Standard_NC64as_T4_v3": (4.352, 0.870),
226
+ "Standard_NC8ads_A10_v4": (0.91, 0.18),
227
+ "Standard_NC16ads_A10_v4": (1.81, 0.36),
228
+ "Standard_NC32ads_A10_v4": (3.62, 0.72),
229
+ "Standard_NC24ads_A100_v4": (3.673, 0.735),
230
+ "Standard_NC48ads_A100_v4": (7.346, 1.470),
231
+ "Standard_NC96ads_A100_v4": (14.692, 2.940),
232
+ "Standard_NC40ads_H100_v5": (6.98, 1.40),
233
+ "Standard_NC80adis_H100_v5": (13.96, 2.80),
234
+ "Standard_NCC40ads_H100_v5": (7.50, 1.50),
235
+ "Standard_ND40rs_v2": (22.0, 6.6),
236
+ "Standard_ND96asr_v4": (27.20, 8.16),
237
+ "Standard_ND96amsr_A100_v4": (32.77, 9.83),
238
+ "Standard_ND96is_H100_v5": (84.00, 25.20),
239
+ "Standard_ND96isr_H100_v5": (98.32, 29.50),
240
+ "Standard_ND96isr_H200_v5": (110.00, 33.00),
241
+ "Standard_ND96isr_MI300X_v5": (60.00, 18.00),
242
+ "Standard_ND96isr_B200_v6": (176.00, 52.80),
243
+ "Standard_ND96isr_GB200_v6": (220.00, 66.00),
244
+ "Standard_ND72isr_GB200_v6": (165.00, 49.50),
245
+ "Standard_NV6ads_A10_v5": (0.45, 0.09),
246
+ "Standard_NV12ads_A10_v5": (0.90, 0.18),
247
+ "Standard_NV18ads_A10_v5": (1.35, 0.27),
248
+ "Standard_NV36ads_A10_v5": (2.70, 0.54),
249
+ "Standard_NV36adms_A10_v5": (4.10, 0.82),
250
+ "Standard_NV72ads_A10_v5": (5.40, 1.08),
251
+ "Standard_NV4ads_V710_v5": (0.34, 0.07),
252
+ "Standard_NV8ads_V710_v5": (0.68, 0.14),
253
+ "Standard_NV12ads_V710_v5": (1.02, 0.21),
254
+ "Standard_NV24ads_V710_v5": (2.04, 0.41),
255
+ "Standard_NV28adms_V710_v5": (3.06, 0.61),
256
+ }