sagemaker-hyperpod 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sagemaker/__init__.py +0 -0
- sagemaker/hyperpod/__init__.py +2 -0
- sagemaker/hyperpod/cli/__init__.py +0 -0
- sagemaker/hyperpod/cli/clients/__init__.py +12 -0
- sagemaker/hyperpod/cli/clients/kubernetes_client.py +365 -0
- sagemaker/hyperpod/cli/commands/__init__.py +0 -0
- sagemaker/hyperpod/cli/commands/cluster.py +627 -0
- sagemaker/hyperpod/cli/commands/inference.py +659 -0
- sagemaker/hyperpod/cli/commands/training.py +347 -0
- sagemaker/hyperpod/cli/constants/__init__.py +0 -0
- sagemaker/hyperpod/cli/constants/command_constants.py +105 -0
- sagemaker/hyperpod/cli/constants/exception_constants.py +13 -0
- sagemaker/hyperpod/cli/constants/hyperpod_instance_types.py +100 -0
- sagemaker/hyperpod/cli/constants/kueue_constants.py +17 -0
- sagemaker/hyperpod/cli/constants/pytorch_constants.py +15 -0
- sagemaker/hyperpod/cli/hyp_cli.py +132 -0
- sagemaker/hyperpod/cli/inference_utils.py +121 -0
- sagemaker/hyperpod/cli/service/__init__.py +12 -0
- sagemaker/hyperpod/cli/service/cancel_training_job.py +70 -0
- sagemaker/hyperpod/cli/service/discover_namespaces.py +108 -0
- sagemaker/hyperpod/cli/service/exec_command.py +75 -0
- sagemaker/hyperpod/cli/service/get_logs.py +136 -0
- sagemaker/hyperpod/cli/service/get_namespaces.py +59 -0
- sagemaker/hyperpod/cli/service/get_training_job.py +110 -0
- sagemaker/hyperpod/cli/service/list_pods.py +138 -0
- sagemaker/hyperpod/cli/service/list_training_jobs.py +178 -0
- sagemaker/hyperpod/cli/service/self_subject_access_review.py +47 -0
- sagemaker/hyperpod/cli/telemetry/__init__.py +12 -0
- sagemaker/hyperpod/cli/telemetry/user_agent.py +36 -0
- sagemaker/hyperpod/cli/templates/__init__.py +12 -0
- sagemaker/hyperpod/cli/templates/k8s_pytorch_job_template.py +74 -0
- sagemaker/hyperpod/cli/training_utils.py +200 -0
- sagemaker/hyperpod/cli/utils.py +183 -0
- sagemaker/hyperpod/cli/validators/__init__.py +12 -0
- sagemaker/hyperpod/cli/validators/cluster_validator.py +52 -0
- sagemaker/hyperpod/cli/validators/job_validator.py +346 -0
- sagemaker/hyperpod/cli/validators/validator.py +75 -0
- sagemaker/hyperpod/common/__init__.py +0 -0
- sagemaker/hyperpod/common/config/__init__.py +1 -0
- sagemaker/hyperpod/common/config/metadata.py +18 -0
- sagemaker/hyperpod/common/utils.py +299 -0
- sagemaker/hyperpod/inference/__init__.py +11 -0
- sagemaker/hyperpod/inference/config/__init__.py +0 -0
- sagemaker/hyperpod/inference/config/constants.py +18 -0
- sagemaker/hyperpod/inference/config/hp_endpoint_config.py +767 -0
- sagemaker/hyperpod/inference/config/hp_jumpstart_endpoint_config.py +572 -0
- sagemaker/hyperpod/inference/hp_endpoint.py +203 -0
- sagemaker/hyperpod/inference/hp_endpoint_base.py +223 -0
- sagemaker/hyperpod/inference/hp_jumpstart_endpoint.py +232 -0
- sagemaker/hyperpod/observability/MonitoringConfig.py +10 -0
- sagemaker/hyperpod/observability/__init__.py +0 -0
- sagemaker/hyperpod/observability/constants.py +2 -0
- sagemaker/hyperpod/observability/utils.py +64 -0
- sagemaker/hyperpod/training/__init__.py +9 -0
- sagemaker/hyperpod/training/config/__init__.py +1 -0
- sagemaker/hyperpod/training/config/hyperpod_pytorch_job_config.py +2977 -0
- sagemaker/hyperpod/training/config/hyperpod_pytorch_job_status.py +3083 -0
- sagemaker/hyperpod/training/hyperpod_pytorch_job.py +253 -0
- sagemaker_hyperpod-3.0.0.dist-info/METADATA +534 -0
- sagemaker_hyperpod-3.0.0.dist-info/RECORD +66 -0
- sagemaker_hyperpod-3.0.0.dist-info/WHEEL +5 -0
- sagemaker_hyperpod-3.0.0.dist-info/entry_points.txt +2 -0
- sagemaker_hyperpod-3.0.0.dist-info/licenses/LICENSE +175 -0
- sagemaker_hyperpod-3.0.0.dist-info/licenses/NOTICE +1 -0
- sagemaker_hyperpod-3.0.0.dist-info/top_level.txt +1 -0
- sagemaker_hyperpod-3.0.0.dist-info/zip-safe +1 -0
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import click
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import yaml
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from sagemaker.hyperpod.training.hyperpod_pytorch_job import HyperPodPytorchJob
|
|
9
|
+
from sagemaker.hyperpod.common.config import Metadata
|
|
10
|
+
import tempfile
|
|
11
|
+
from typing import List, Dict, Any, Optional, Callable, get_args, get_origin, Literal
|
|
12
|
+
from sagemaker.hyperpod.cli.training_utils import generate_click_command
|
|
13
|
+
from importlib.metadata import entry_points
|
|
14
|
+
from hyperpod_pytorch_job_template.registry import SCHEMA_REGISTRY
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@click.command("hyp-pytorch-job")
|
|
18
|
+
@click.option("--version", default="1.0", help="Schema version to use")
|
|
19
|
+
@click.option("--debug", default=False, help="Enable debug mode")
|
|
20
|
+
@generate_click_command(
|
|
21
|
+
schema_pkg="hyperpod_pytorch_job_template",
|
|
22
|
+
registry=SCHEMA_REGISTRY,
|
|
23
|
+
)
|
|
24
|
+
def pytorch_create(version, debug, config):
|
|
25
|
+
"""Create a PyTorch job"""
|
|
26
|
+
try:
|
|
27
|
+
click.echo(f"Using version: {version}")
|
|
28
|
+
job_name = config.get("name")
|
|
29
|
+
namespace = config.get("namespace")
|
|
30
|
+
spec = config.get("spec")
|
|
31
|
+
|
|
32
|
+
# Prepare metadata
|
|
33
|
+
metadata_kwargs = {"name": job_name}
|
|
34
|
+
if namespace:
|
|
35
|
+
metadata_kwargs["namespace"] = namespace
|
|
36
|
+
|
|
37
|
+
# Prepare job kwargs
|
|
38
|
+
job_kwargs = {
|
|
39
|
+
"metadata": Metadata(**metadata_kwargs),
|
|
40
|
+
"replica_specs": spec.get("replica_specs"),
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
# Add nproc_per_node if present
|
|
44
|
+
if "nproc_per_node" in spec:
|
|
45
|
+
job_kwargs["nproc_per_node"] = spec.get("nproc_per_node")
|
|
46
|
+
|
|
47
|
+
# Add run_policy if present
|
|
48
|
+
if "run_policy" in spec:
|
|
49
|
+
job_kwargs["run_policy"] = spec.get("run_policy")
|
|
50
|
+
|
|
51
|
+
# Create job
|
|
52
|
+
job = HyperPodPytorchJob(**job_kwargs)
|
|
53
|
+
job.create(debug=debug)
|
|
54
|
+
|
|
55
|
+
except Exception as e:
|
|
56
|
+
raise click.UsageError(f"Failed to create job: {str(e)}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@click.command("hyp-pytorch-job")
|
|
60
|
+
@click.option(
|
|
61
|
+
"--namespace",
|
|
62
|
+
"-n",
|
|
63
|
+
default="default",
|
|
64
|
+
help="Optional. The namespace to list jobs from. Defaults to 'default' namespace.",
|
|
65
|
+
)
|
|
66
|
+
def list_jobs(namespace: str):
|
|
67
|
+
"""List all HyperPod PyTorch jobs"""
|
|
68
|
+
try:
|
|
69
|
+
jobs = HyperPodPytorchJob.list(namespace=namespace)
|
|
70
|
+
|
|
71
|
+
if not jobs:
|
|
72
|
+
click.echo("No jobs found.")
|
|
73
|
+
return
|
|
74
|
+
|
|
75
|
+
# Define headers and widths
|
|
76
|
+
headers = ["NAME", "NAMESPACE", "STATUS", "AGE"]
|
|
77
|
+
widths = [30, 20, 15, 15]
|
|
78
|
+
|
|
79
|
+
# Print header
|
|
80
|
+
header = "".join(f"{h:<{w}}" for h, w in zip(headers, widths))
|
|
81
|
+
click.echo("\n" + header)
|
|
82
|
+
click.echo("-" * sum(widths))
|
|
83
|
+
|
|
84
|
+
# Print each job
|
|
85
|
+
for job in jobs:
|
|
86
|
+
# Get status from conditions
|
|
87
|
+
status = "Unknown"
|
|
88
|
+
age = "N/A"
|
|
89
|
+
if job.status and job.status.conditions:
|
|
90
|
+
for condition in reversed(job.status.conditions):
|
|
91
|
+
if condition.status == "True":
|
|
92
|
+
status = condition.type
|
|
93
|
+
break
|
|
94
|
+
|
|
95
|
+
# Calculate age
|
|
96
|
+
if job.status and job.status.conditions:
|
|
97
|
+
# Find the 'Created' condition to get the start time
|
|
98
|
+
created_condition = next(
|
|
99
|
+
(c for c in job.status.conditions if c.type == "Created"), None
|
|
100
|
+
)
|
|
101
|
+
if created_condition and created_condition.lastTransitionTime:
|
|
102
|
+
from datetime import datetime, timezone
|
|
103
|
+
|
|
104
|
+
start_time = datetime.fromisoformat(
|
|
105
|
+
created_condition.lastTransitionTime.replace("Z", "+00:00")
|
|
106
|
+
)
|
|
107
|
+
now = datetime.now(timezone.utc)
|
|
108
|
+
delta = now - start_time
|
|
109
|
+
if delta.days > 0:
|
|
110
|
+
age = f"{delta.days}d"
|
|
111
|
+
else:
|
|
112
|
+
hours = delta.seconds // 3600
|
|
113
|
+
if hours > 0:
|
|
114
|
+
age = f"{hours}h"
|
|
115
|
+
else:
|
|
116
|
+
minutes = (delta.seconds % 3600) // 60
|
|
117
|
+
age = f"{minutes}m"
|
|
118
|
+
|
|
119
|
+
# Format row
|
|
120
|
+
row = "".join(
|
|
121
|
+
[
|
|
122
|
+
f"{job.metadata.name:<{widths[0]}}",
|
|
123
|
+
f"{job.metadata.namespace:<{widths[1]}}",
|
|
124
|
+
f"{status:<{widths[2]}}",
|
|
125
|
+
f"{age:<{widths[3]}}",
|
|
126
|
+
]
|
|
127
|
+
)
|
|
128
|
+
click.echo(row)
|
|
129
|
+
|
|
130
|
+
click.echo() # Add empty line at the end
|
|
131
|
+
|
|
132
|
+
except Exception as e:
|
|
133
|
+
raise click.UsageError(f"Failed to list jobs: {str(e)}")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@click.command("hyp-pytorch-job")
|
|
137
|
+
@click.option(
|
|
138
|
+
"--job-name", required=True, help="Required. The name of the job to describe"
|
|
139
|
+
)
|
|
140
|
+
@click.option(
|
|
141
|
+
"--namespace",
|
|
142
|
+
"-n",
|
|
143
|
+
default="default",
|
|
144
|
+
help="Optional. The namespace of the job. Defaults to 'default' namespace.",
|
|
145
|
+
)
|
|
146
|
+
def pytorch_describe(job_name: str, namespace: str):
|
|
147
|
+
"""Describe a HyperPod PyTorch job"""
|
|
148
|
+
try:
|
|
149
|
+
job = HyperPodPytorchJob.get(name=job_name, namespace=namespace)
|
|
150
|
+
|
|
151
|
+
if job is None:
|
|
152
|
+
raise click.UsageError(f"Job {job_name} not found in namespace {namespace}")
|
|
153
|
+
|
|
154
|
+
# Print basic info
|
|
155
|
+
click.echo("\nJob Details:")
|
|
156
|
+
click.echo("=" * 80)
|
|
157
|
+
click.echo(f"Name: {job.metadata.name}")
|
|
158
|
+
click.echo(f"Namespace: {job.metadata.namespace}")
|
|
159
|
+
|
|
160
|
+
# Print Spec details
|
|
161
|
+
click.echo("\nSpec:")
|
|
162
|
+
click.echo("-" * 80)
|
|
163
|
+
click.echo(f"Processes per Node: {getattr(job, 'nprocPerNode', 'N/A')}")
|
|
164
|
+
|
|
165
|
+
# Print Replica Specs
|
|
166
|
+
for replica in job.replicaSpecs:
|
|
167
|
+
click.echo(f"\nReplica Spec:")
|
|
168
|
+
click.echo(f" Name: {getattr(replica, 'name', 'N/A')}")
|
|
169
|
+
click.echo(f" Replicas: {getattr(replica, 'replicas', 'N/A')}")
|
|
170
|
+
click.echo(f" Spares: {getattr(replica, 'spares', 'N/A')}")
|
|
171
|
+
|
|
172
|
+
# Container details
|
|
173
|
+
if (
|
|
174
|
+
hasattr(replica, "template")
|
|
175
|
+
and hasattr(replica.template, "spec")
|
|
176
|
+
and hasattr(replica.template.spec, "containers")
|
|
177
|
+
):
|
|
178
|
+
for container in replica.template.spec.containers:
|
|
179
|
+
click.echo("\n Container:")
|
|
180
|
+
click.echo(
|
|
181
|
+
f" Name: {getattr(container, 'name', 'N/A')}"
|
|
182
|
+
)
|
|
183
|
+
click.echo(
|
|
184
|
+
f" Image: {getattr(container, 'image', 'N/A')}"
|
|
185
|
+
)
|
|
186
|
+
click.echo(
|
|
187
|
+
f" Image Pull Policy: {getattr(container, 'imagePullPolicy', 'N/A')}"
|
|
188
|
+
)
|
|
189
|
+
if container.resources:
|
|
190
|
+
click.echo(" Resources:")
|
|
191
|
+
if container.resources.limits:
|
|
192
|
+
click.echo(f" Limits: {container.resources.limits}")
|
|
193
|
+
if container.resources.requests:
|
|
194
|
+
click.echo(
|
|
195
|
+
f" Requests: {container.resources.requests}"
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# Print Run Policy
|
|
199
|
+
click.echo("\nRun Policy:")
|
|
200
|
+
click.echo("-" * 80)
|
|
201
|
+
if hasattr(job, "runPolicy"):
|
|
202
|
+
click.echo(
|
|
203
|
+
f"Clean Pod Policy: {getattr(job.runPolicy, 'cleanPodPolicy', 'N/A')}"
|
|
204
|
+
)
|
|
205
|
+
click.echo(
|
|
206
|
+
f"TTL Seconds After Finished: {getattr(job.runPolicy, 'ttlSecondsAfterFinished', 'N/A')}"
|
|
207
|
+
)
|
|
208
|
+
else:
|
|
209
|
+
click.echo("Run Policy: N/A")
|
|
210
|
+
|
|
211
|
+
# Print Status
|
|
212
|
+
click.echo("\nStatus:")
|
|
213
|
+
click.echo("-" * 80)
|
|
214
|
+
if job.status:
|
|
215
|
+
if job.status.conditions:
|
|
216
|
+
click.echo("Conditions:")
|
|
217
|
+
for condition in job.status.conditions:
|
|
218
|
+
click.echo(
|
|
219
|
+
f" Type: {getattr(condition, 'type', 'N/A')}"
|
|
220
|
+
)
|
|
221
|
+
click.echo(
|
|
222
|
+
f" Status: {getattr(condition, 'status', 'N/A')}"
|
|
223
|
+
)
|
|
224
|
+
click.echo(
|
|
225
|
+
f" Last Transition: {getattr(condition, 'lastTransitionTime', 'N/A')}"
|
|
226
|
+
)
|
|
227
|
+
if condition.message:
|
|
228
|
+
click.echo(f" Message: {condition.message}")
|
|
229
|
+
click.echo()
|
|
230
|
+
else:
|
|
231
|
+
click.echo("No status information available")
|
|
232
|
+
|
|
233
|
+
except Exception as e:
|
|
234
|
+
raise click.UsageError(f"Failed to describe job: {str(e)}")
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@click.command("hyp-pytorch-job")
|
|
238
|
+
@click.option(
|
|
239
|
+
"--job-name", required=True, help="Required. The name of the job to delete"
|
|
240
|
+
)
|
|
241
|
+
@click.option(
|
|
242
|
+
"--namespace",
|
|
243
|
+
"-n",
|
|
244
|
+
default="default",
|
|
245
|
+
help="Optional. The namespace of the job. Defaults to 'default' namespace.",
|
|
246
|
+
)
|
|
247
|
+
def pytorch_delete(job_name: str, namespace: str):
|
|
248
|
+
"""Delete a HyperPod PyTorch job"""
|
|
249
|
+
try:
|
|
250
|
+
job = HyperPodPytorchJob.get(name=job_name, namespace=namespace)
|
|
251
|
+
job.delete()
|
|
252
|
+
|
|
253
|
+
if job is None:
|
|
254
|
+
raise click.UsageError(f"Job {job_name} not found in namespace {namespace}")
|
|
255
|
+
|
|
256
|
+
except Exception as e:
|
|
257
|
+
raise click.UsageError(f"Failed to describe job: {str(e)}")
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@click.command("hyp-pytorch-job")
|
|
261
|
+
@click.option(
|
|
262
|
+
"--job-name",
|
|
263
|
+
required=True,
|
|
264
|
+
help="Required. Specify the job name to list its associated pods.",
|
|
265
|
+
)
|
|
266
|
+
@click.option(
|
|
267
|
+
"--namespace",
|
|
268
|
+
"-n",
|
|
269
|
+
default="default",
|
|
270
|
+
help="Optional. The namespace of the job. Defaults to 'default' namespace.",
|
|
271
|
+
)
|
|
272
|
+
def pytorch_list_pods(job_name: str, namespace: str):
|
|
273
|
+
"""List all HyperPod PyTorch pods corresponding to the job"""
|
|
274
|
+
try:
|
|
275
|
+
job = HyperPodPytorchJob.get(name=job_name, namespace=namespace)
|
|
276
|
+
pods = job.list_pods()
|
|
277
|
+
|
|
278
|
+
if not pods:
|
|
279
|
+
click.echo(f"\nNo pods found for job: {job_name}")
|
|
280
|
+
return
|
|
281
|
+
|
|
282
|
+
# Define headers and widths
|
|
283
|
+
headers = ["POD NAME", "NAMESPACE"]
|
|
284
|
+
widths = [50, 20]
|
|
285
|
+
|
|
286
|
+
# Print header
|
|
287
|
+
click.echo(f"\nPods for job: {job_name}")
|
|
288
|
+
header = "".join(f"{h:<{w}}" for h, w in zip(headers, widths))
|
|
289
|
+
click.echo("\n" + header)
|
|
290
|
+
click.echo("-" * sum(widths))
|
|
291
|
+
|
|
292
|
+
# Print each pod
|
|
293
|
+
for pod in pods:
|
|
294
|
+
row = "".join([f"{pod:<{widths[0]}}", f"{namespace:<{widths[1]}}"])
|
|
295
|
+
click.echo(row)
|
|
296
|
+
|
|
297
|
+
click.echo()
|
|
298
|
+
|
|
299
|
+
except Exception as e:
|
|
300
|
+
raise click.UsageError(f"Failed to list jobs: {str(e)}")
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@click.command("hyp-pytorch-job")
|
|
304
|
+
@click.option(
|
|
305
|
+
"--job-name",
|
|
306
|
+
required=True,
|
|
307
|
+
help="Required. Specify the job name for pod log retrieval.",
|
|
308
|
+
)
|
|
309
|
+
@click.option(
|
|
310
|
+
"--pod-name", required=True, help="Required. The name of the pod to get logs from."
|
|
311
|
+
)
|
|
312
|
+
@click.option(
|
|
313
|
+
"--namespace",
|
|
314
|
+
"-n",
|
|
315
|
+
default="default",
|
|
316
|
+
help="Optional. The namespace of the job. Defaults to 'default' namespace.",
|
|
317
|
+
)
|
|
318
|
+
def pytorch_get_logs(job_name: str, pod_name: str, namespace: str):
|
|
319
|
+
"""Get specific logs from pod corresponding to the job"""
|
|
320
|
+
try:
|
|
321
|
+
click.echo("Listing logs for pod: " + pod_name)
|
|
322
|
+
job = HyperPodPytorchJob.get(name=job_name, namespace=namespace)
|
|
323
|
+
logs = job.get_logs_from_pod(pod_name=pod_name)
|
|
324
|
+
|
|
325
|
+
if not logs:
|
|
326
|
+
click.echo("No logs available.")
|
|
327
|
+
return
|
|
328
|
+
|
|
329
|
+
# Split logs into lines and display them
|
|
330
|
+
log_lines = logs.split("\n")
|
|
331
|
+
for line in log_lines:
|
|
332
|
+
if line.strip(): # Skip empty lines
|
|
333
|
+
# Color coding based on log level
|
|
334
|
+
if "ERROR" in line.upper():
|
|
335
|
+
click.secho(line, fg="red")
|
|
336
|
+
elif "WARNING" in line.upper():
|
|
337
|
+
click.secho(line, fg="yellow")
|
|
338
|
+
elif "INFO" in line.upper():
|
|
339
|
+
click.secho(line, fg="green")
|
|
340
|
+
else:
|
|
341
|
+
click.echo(line)
|
|
342
|
+
|
|
343
|
+
click.echo("\nEnd of logs")
|
|
344
|
+
click.echo("=" * 80)
|
|
345
|
+
|
|
346
|
+
except Exception as e:
|
|
347
|
+
raise click.UsageError(f"Failed to list jobs: {str(e)}")
|
|
File without changes
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
4
|
+
# may not use this file except in compliance with the License. A copy of
|
|
5
|
+
# the License is located at
|
|
6
|
+
#
|
|
7
|
+
# http://aws.amazon.com/apache2.0/
|
|
8
|
+
#
|
|
9
|
+
# or in the "license" file accompanying this file. This file is
|
|
10
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
11
|
+
# ANY KIND, either express or implied. See the License for the specific
|
|
12
|
+
# language governing permissions and limitations under the License.
|
|
13
|
+
from enum import Enum
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
GENERATED_LAUNCHER_CONFIG_FILE_PATH = "/tmp/"
|
|
17
|
+
HYPERPOD_KUBERNETES_JOB_PREFIX = "hyperpod-k8s-job"
|
|
18
|
+
HYPERPOD_CLUSTER_CONTEXT_FILE_NAME = "hyperpod_current_context.json"
|
|
19
|
+
NODE_AFFINITY_DICT = {
|
|
20
|
+
"required": {"sagemaker.amazonaws.com/node-health-status": ["Schedulable"]},
|
|
21
|
+
"preferred": {"sagemaker.amazonaws.com/deep-health-check-status": ["Passed"]},
|
|
22
|
+
"weights": [100],
|
|
23
|
+
}
|
|
24
|
+
DEEP_HEALTH_CHECK_PASSED_ONLY_NODE_AFFINITY_DICT = {
|
|
25
|
+
"required": {
|
|
26
|
+
"sagemaker.amazonaws.com/deep-health-check-status": ["Passed"],
|
|
27
|
+
},
|
|
28
|
+
}
|
|
29
|
+
KUEUE_QUEUE_NAME_LABEL_KEY = "kueue.x-k8s.io/queue-name"
|
|
30
|
+
KUEUE_WORKLOAD_PRIORITY_CLASS_LABEL_KEY = "kueue.x-k8s.io/priority-class"
|
|
31
|
+
KUEUE_JOB_UID_LABEL_KEY = "kueue.x-k8s.io/job-uid"
|
|
32
|
+
HYPERPOD_AUTO_RESUME_ANNOTATION_KEY = "sagemaker.amazonaws.com/enable-job-auto-resume"
|
|
33
|
+
HYPERPOD_MAX_RETRY_ANNOTATION_KEY = "sagemaker.amazonaws.com/job-max-retry-count"
|
|
34
|
+
ENV_VARS_DICT = {"NCCL_DEBUG": "INFO"}
|
|
35
|
+
SAGEMAKER_HYPERPOD_NAME_LABEL = "sagemaker.amazonaws.com/cluster-name"
|
|
36
|
+
HP_HEALTH_STATUS_LABEL = "sagemaker.amazonaws.com/node-health-status"
|
|
37
|
+
INSTANCE_TYPE_LABEL = "node.kubernetes.io/instance-type"
|
|
38
|
+
DEEP_HEALTH_CHECK_STATUS_LABEL = "sagemaker.amazonaws.com/deep-health-check-status"
|
|
39
|
+
SAGEMAKER_MANAGED_QUEUE_LABEL= "sagemaker.amazonaws.com/sagemaker-managed-queue"
|
|
40
|
+
SAGEMAKER_QUOTA_ALLOCATION_LABEL = "sagemaker.amazonaws.com/quota-allocation-id"
|
|
41
|
+
TEMP_KUBE_CONFIG_FILE = "/tmp/kubeconfig"
|
|
42
|
+
HYPERPOD_NAMESPACE_PREFIX = "hyperpod-ns-"
|
|
43
|
+
SAGEMAKER_MANAGED_LOCAL_QUEUE_SUFFIX = "-localqueue"
|
|
44
|
+
SAGEMAKER_MANAGED_CLUSTER_QUEUE_SUFFIX = "-clusterqueue"
|
|
45
|
+
SAGEMAKER_TRAINING_LAUNCHER_DIR = str(Path(__file__).parent.parent / "sagemaker_hyperpod_recipes")
|
|
46
|
+
NVIDIA_GPU_RESOURCE_LIMIT_KEY = "nvidia.com/gpu"
|
|
47
|
+
AVAILABLE_ACCELERATOR_DEVICES_KEY = "AvailableAcceleratorDevices"
|
|
48
|
+
TOTAL_ACCELERATOR_DEVICES_KEY = "TotalAcceleratorDevices"
|
|
49
|
+
USER_NAME_LABEL_KEY = "sagemaker.user/created-by"
|
|
50
|
+
|
|
51
|
+
class PullPolicy(Enum):
|
|
52
|
+
ALWAYS = "Always"
|
|
53
|
+
IF_NOT_PRESENT = "IfNotPresent"
|
|
54
|
+
NEVER = "Never"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class RestartPolicy(Enum):
|
|
58
|
+
ALWAYS = "Always"
|
|
59
|
+
ON_FAILURE = "OnFailure"
|
|
60
|
+
NEVER = "Never"
|
|
61
|
+
EXIT_CODE = "ExitCode"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class OutputFormat(Enum):
|
|
65
|
+
JSON = "json"
|
|
66
|
+
TABLE = "table"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class PersistentVolumeClaim:
|
|
70
|
+
claim_name: str
|
|
71
|
+
mount_path: str
|
|
72
|
+
|
|
73
|
+
def __init__(self, claim_name, mount_path):
|
|
74
|
+
self.claim_name = claim_name
|
|
75
|
+
self.mount_path = mount_path
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class SchedulerType(Enum):
|
|
79
|
+
KUEUE = "Kueue"
|
|
80
|
+
SAGEMAKER = "SageMaker"
|
|
81
|
+
NONE = "None"
|
|
82
|
+
|
|
83
|
+
def get_values():
|
|
84
|
+
return [scheduler_type.value for scheduler_type in SchedulerType]
|
|
85
|
+
|
|
86
|
+
def get_default():
|
|
87
|
+
return SchedulerType.SAGEMAKER
|
|
88
|
+
|
|
89
|
+
class JobPatchType(Enum):
|
|
90
|
+
SUSPEND = "suspend"
|
|
91
|
+
UNSUSPEND = "unsuspend"
|
|
92
|
+
|
|
93
|
+
def get_values():
|
|
94
|
+
return [type.value for type in JobPatchType]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class Volume:
|
|
98
|
+
volume_name: str
|
|
99
|
+
host_path: str
|
|
100
|
+
mount_path: str
|
|
101
|
+
|
|
102
|
+
def __init__(self, volume_name, host_path, mount_path):
|
|
103
|
+
self.host_path = host_path
|
|
104
|
+
self.mount_path = mount_path
|
|
105
|
+
self.volume_name = volume_name
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
4
|
+
# may not use this file except in compliance with the License. A copy of
|
|
5
|
+
# the License is located at
|
|
6
|
+
#
|
|
7
|
+
# http://aws.amazon.com/apache2.0/
|
|
8
|
+
#
|
|
9
|
+
# or in the "license" file accompanying this file. This file is
|
|
10
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
11
|
+
# ANY KIND, either express or implied. See the License for the specific
|
|
12
|
+
# language governing permissions and limitations under the License.
|
|
13
|
+
RESOURCE_NOT_FOUND_CODE = 404
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
4
|
+
# may not use this file except in compliance with the License. A copy of
|
|
5
|
+
# the License is located at
|
|
6
|
+
#
|
|
7
|
+
# http://aws.amazon.com/apache2.0/
|
|
8
|
+
#
|
|
9
|
+
# or in the "license" file accompanying this file. This file is
|
|
10
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
11
|
+
# ANY KIND, either express or implied. See the License for the specific
|
|
12
|
+
# language governing permissions and limitations under the License.
|
|
13
|
+
from enum import Enum
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class HyperpodInstanceType(Enum):
|
|
17
|
+
ML_C5_LARGE = "ml.c5.large"
|
|
18
|
+
ML_C5_XLARGE = "ml.c5.xlarge"
|
|
19
|
+
ML_C5_2XLARGE = "ml.c5.2xlarge"
|
|
20
|
+
ML_C5_4XLARGE = "ml.c5.4xlarge"
|
|
21
|
+
ML_C5_9XLARGE = "ml.c5.9xlarge"
|
|
22
|
+
ML_C5_12XLARGE = "ml.c5.12xlarge"
|
|
23
|
+
ML_C5_18XLARGE = "ml.c5.18xlarge"
|
|
24
|
+
ML_C5_24XLARGE = "ml.c5.24xlarge"
|
|
25
|
+
ML_C5N_LARGE = "ml.c5n.large"
|
|
26
|
+
ML_C5N_2XLARGE = "ml.c5n.2xlarge"
|
|
27
|
+
ML_C5N_4XLARGE = "ml.c5n.4xlarge"
|
|
28
|
+
ML_C5N_9XLARGE = "ml.c5n.9xlarge"
|
|
29
|
+
ML_C5N_18XLARGE = "ml.c5n.18xlarge"
|
|
30
|
+
ML_G5_XLARGE = "ml.g5.xlarge"
|
|
31
|
+
ML_G5_2XLARGE = "ml.g5.2xlarge"
|
|
32
|
+
ML_G5_4XLARGE = "ml.g5.4xlarge"
|
|
33
|
+
ML_G5_8XLARGE = "ml.g5.8xlarge"
|
|
34
|
+
ML_G5_12XLARGE = "ml.g5.12xlarge"
|
|
35
|
+
ML_G5_16XLARGE = "ml.g5.16xlarge"
|
|
36
|
+
ML_G5_24XLARGE = "ml.g5.24xlarge"
|
|
37
|
+
ML_G5_48XLARGE = "ml.g5.48xlarge"
|
|
38
|
+
ML_G6_XLARGE = "ml.g6.xlarge"
|
|
39
|
+
ML_G6_2XLARGE = "ml.g6.2xlarge"
|
|
40
|
+
ML_G6_4XLARGE = "ml.g6.4xlarge"
|
|
41
|
+
ML_G6_8XLARGE = "ml.g6.8xlarge"
|
|
42
|
+
ML_G6_12XLARGE = "ml.g6.12xlarge"
|
|
43
|
+
ML_G6_16XLARGE = "ml.g6.16xlarge"
|
|
44
|
+
ML_G6_24XLARGE = "ml.g6.24xlarge"
|
|
45
|
+
ML_G6_48XLARGE = "ml.g6.48xlarge"
|
|
46
|
+
ML_G6E_XLARGE = "ml.g6e.xlarge"
|
|
47
|
+
ML_G6E_2XLARGE = "ml.g6e.2xlarge"
|
|
48
|
+
ML_G6E_4XLARGE = "ml.g6e.4xlarge"
|
|
49
|
+
ML_G6E_8XLARGE = "ml.g6e.8xlarge"
|
|
50
|
+
ML_G6E_12XLARGE = "ml.g6e.12xlarge"
|
|
51
|
+
ML_G6E_16XLARGE = "ml.g6e.16xlarge"
|
|
52
|
+
ML_G6E_24XLARGE = "ml.g6e.24xlarge"
|
|
53
|
+
ML_G6E_48XLARGE = "ml.g6e.48xlarge"
|
|
54
|
+
ML_GR6_4XLARGE = "ml.gr6.4xlarge"
|
|
55
|
+
ML_GR6_8XLARGE = "ml.gr6.8xlarge"
|
|
56
|
+
ML_I3EN_LARGE = "ml.i3en.large"
|
|
57
|
+
ML_I3EN_XLARGE = "ml.i3en.xlarge"
|
|
58
|
+
ML_I3EN_2XLARGE = "ml.i3en.2xlarge"
|
|
59
|
+
ML_I3EN_3XLARGE = "ml.i3en.3xlarge"
|
|
60
|
+
ML_I3EN_6XLARGE = "ml.i3en.6xlarge"
|
|
61
|
+
ML_I3EN_12XLARGE = "ml.i3en.12xlarge"
|
|
62
|
+
ML_I3EN_24XLARGE = "ml.i3en.24xlarge"
|
|
63
|
+
ML_M5_LARGE = "ml.m5.large"
|
|
64
|
+
ML_M5_XLARGE = "ml.m5.xlarge"
|
|
65
|
+
ML_M5_2XLARGE = "ml.m5.2xlarge"
|
|
66
|
+
ML_M5_4XLARGE = "ml.m5.4xlarge"
|
|
67
|
+
ML_M5_8XLARGE = "ml.m5.8xlarge"
|
|
68
|
+
ML_M5_12XLARGE = "ml.m5.12xlarge"
|
|
69
|
+
ML_M5_16XLARGE = "ml.m5.16xlarge"
|
|
70
|
+
ML_M5_24XLARGE = "ml.m5.24xlarge"
|
|
71
|
+
ML_M7I_LARGE = "ml.m7i.large"
|
|
72
|
+
ML_M7I_XLARGE = "ml.m7i.xlarge"
|
|
73
|
+
ML_M7I_2XLARGE = "ml.m7i.2xlarge"
|
|
74
|
+
ML_M7I_4XLARGE = "ml.m7i.4xlarge"
|
|
75
|
+
ML_M7I_8XLARGE = "ml.m7i.8xlarge"
|
|
76
|
+
ML_M7I_12XLARGE = "ml.m7i.12xlarge"
|
|
77
|
+
ML_M7I_16XLARGE = "ml.m7i.16xlarge"
|
|
78
|
+
ML_M7I_24XLARGE = "ml.m7i.24xlarge"
|
|
79
|
+
ML_M7I_48XLARGE = "ml.m7i.48xlarge"
|
|
80
|
+
ML_P4D_24XLARGE = "ml.p4d.24xlarge"
|
|
81
|
+
ML_P4DE_24XLARGE = "ml.p4de.24xlarge"
|
|
82
|
+
ML_P5_48XLARGE = "ml.p5.48xlarge"
|
|
83
|
+
ML_P5E_48XLARGE = "ml.p5e.48xlarge"
|
|
84
|
+
ML_P5EN_48XLARGE = "ml.p5en.48xlarge"
|
|
85
|
+
ML_R7I_LARGE = "ml.r7i.large"
|
|
86
|
+
ML_R7I_XLARGE = "ml.r7i.xlarge"
|
|
87
|
+
ML_R7I_2XLARGE = "ml.r7i.2xlarge"
|
|
88
|
+
ML_R7I_4XLARGE = "ml.r7i.4xlarge"
|
|
89
|
+
ML_R7I_8XLARGE = "ml.r7i.8xlarge"
|
|
90
|
+
ML_R7I_12XLARGE = "ml.r7i.12xlarge"
|
|
91
|
+
ML_R7I_16XLARGE = "ml.r7i.16xlarge"
|
|
92
|
+
ML_R7I_24XLARGE = "ml.r7i.24xlarge"
|
|
93
|
+
ML_R7I_48XLARGE = "ml.r7i.48xlarge"
|
|
94
|
+
ML_T3_MEDIUM = "ml.t3.medium"
|
|
95
|
+
ML_T3_LARGE = "ml.t3.large"
|
|
96
|
+
ML_T3_XLARGE = "ml.t3.xlarge"
|
|
97
|
+
ML_T3_2XLARGE = "ml.t3.2xlarge"
|
|
98
|
+
ML_TRN1_32XLARGE = "ml.trn1.32xlarge"
|
|
99
|
+
ML_TRN1N_32XLARGE = "ml.trn1n.32xlarge"
|
|
100
|
+
ML_TRN2_48XLARGE = "ml.trn2.48xlarge"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
4
|
+
# may not use this file except in compliance with the License. A copy of
|
|
5
|
+
# the License is located at
|
|
6
|
+
#
|
|
7
|
+
# http://aws.amazon.com/apache2.0/
|
|
8
|
+
#
|
|
9
|
+
# or in the "license" file accompanying this file. This file is
|
|
10
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
11
|
+
# ANY KIND, either express or implied. See the License for the specific
|
|
12
|
+
# language governing permissions and limitations under the License.
|
|
13
|
+
KUEUE_CUSTOM_OBJECT_GROUP = "kueue.x-k8s.io"
|
|
14
|
+
WORKLOAD_CUSTOM_OBJECT_PLURAL = "workloads"
|
|
15
|
+
KUEUE_CUSTOM_OBJECT_VERSION = "v1beta1"
|
|
16
|
+
WORKLOAD_PRIORITY_CLASS_CUSTOM_OBJECT_PLURAL = "workloadpriorityclasses"
|
|
17
|
+
CLUSTER_QUEUE_PRIORITY_CLASS_CUSTOM_OBJECT_PLURAL = "clusterqueues"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
|
4
|
+
# may not use this file except in compliance with the License. A copy of
|
|
5
|
+
# the License is located at
|
|
6
|
+
#
|
|
7
|
+
# http://aws.amazon.com/apache2.0/
|
|
8
|
+
#
|
|
9
|
+
# or in the "license" file accompanying this file. This file is
|
|
10
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
|
11
|
+
# ANY KIND, either express or implied. See the License for the specific
|
|
12
|
+
# language governing permissions and limitations under the License.
|
|
13
|
+
PYTORCH_CUSTOM_OBJECT_GROUP = "kubeflow.org"
|
|
14
|
+
PYTORCH_CUSTOM_OBJECT_PLURAL = "pytorchjobs"
|
|
15
|
+
PYTORCH_CUSTOM_OBJECT_VERSION = "v1"
|