sagemaker-hyperpod 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sagemaker/__init__.py +0 -0
- sagemaker/hyperpod/__init__.py +2 -0
- sagemaker/hyperpod/cli/__init__.py +0 -0
- sagemaker/hyperpod/cli/clients/__init__.py +12 -0
- sagemaker/hyperpod/cli/clients/kubernetes_client.py +365 -0
- sagemaker/hyperpod/cli/commands/__init__.py +0 -0
- sagemaker/hyperpod/cli/commands/cluster.py +627 -0
- sagemaker/hyperpod/cli/commands/inference.py +659 -0
- sagemaker/hyperpod/cli/commands/training.py +347 -0
- sagemaker/hyperpod/cli/constants/__init__.py +0 -0
- sagemaker/hyperpod/cli/constants/command_constants.py +105 -0
- sagemaker/hyperpod/cli/constants/exception_constants.py +13 -0
- sagemaker/hyperpod/cli/constants/hyperpod_instance_types.py +100 -0
- sagemaker/hyperpod/cli/constants/kueue_constants.py +17 -0
- sagemaker/hyperpod/cli/constants/pytorch_constants.py +15 -0
- sagemaker/hyperpod/cli/hyp_cli.py +132 -0
- sagemaker/hyperpod/cli/inference_utils.py +121 -0
- sagemaker/hyperpod/cli/service/__init__.py +12 -0
- sagemaker/hyperpod/cli/service/cancel_training_job.py +70 -0
- sagemaker/hyperpod/cli/service/discover_namespaces.py +108 -0
- sagemaker/hyperpod/cli/service/exec_command.py +75 -0
- sagemaker/hyperpod/cli/service/get_logs.py +136 -0
- sagemaker/hyperpod/cli/service/get_namespaces.py +59 -0
- sagemaker/hyperpod/cli/service/get_training_job.py +110 -0
- sagemaker/hyperpod/cli/service/list_pods.py +138 -0
- sagemaker/hyperpod/cli/service/list_training_jobs.py +178 -0
- sagemaker/hyperpod/cli/service/self_subject_access_review.py +47 -0
- sagemaker/hyperpod/cli/telemetry/__init__.py +12 -0
- sagemaker/hyperpod/cli/telemetry/user_agent.py +36 -0
- sagemaker/hyperpod/cli/templates/__init__.py +12 -0
- sagemaker/hyperpod/cli/templates/k8s_pytorch_job_template.py +74 -0
- sagemaker/hyperpod/cli/training_utils.py +200 -0
- sagemaker/hyperpod/cli/utils.py +183 -0
- sagemaker/hyperpod/cli/validators/__init__.py +12 -0
- sagemaker/hyperpod/cli/validators/cluster_validator.py +52 -0
- sagemaker/hyperpod/cli/validators/job_validator.py +346 -0
- sagemaker/hyperpod/cli/validators/validator.py +75 -0
- sagemaker/hyperpod/common/__init__.py +0 -0
- sagemaker/hyperpod/common/config/__init__.py +1 -0
- sagemaker/hyperpod/common/config/metadata.py +18 -0
- sagemaker/hyperpod/common/utils.py +299 -0
- sagemaker/hyperpod/inference/__init__.py +11 -0
- sagemaker/hyperpod/inference/config/__init__.py +0 -0
- sagemaker/hyperpod/inference/config/constants.py +18 -0
- sagemaker/hyperpod/inference/config/hp_endpoint_config.py +767 -0
- sagemaker/hyperpod/inference/config/hp_jumpstart_endpoint_config.py +572 -0
- sagemaker/hyperpod/inference/hp_endpoint.py +203 -0
- sagemaker/hyperpod/inference/hp_endpoint_base.py +223 -0
- sagemaker/hyperpod/inference/hp_jumpstart_endpoint.py +232 -0
- sagemaker/hyperpod/observability/MonitoringConfig.py +10 -0
- sagemaker/hyperpod/observability/__init__.py +0 -0
- sagemaker/hyperpod/observability/constants.py +2 -0
- sagemaker/hyperpod/observability/utils.py +64 -0
- sagemaker/hyperpod/training/__init__.py +9 -0
- sagemaker/hyperpod/training/config/__init__.py +1 -0
- sagemaker/hyperpod/training/config/hyperpod_pytorch_job_config.py +2977 -0
- sagemaker/hyperpod/training/config/hyperpod_pytorch_job_status.py +3083 -0
- sagemaker/hyperpod/training/hyperpod_pytorch_job.py +253 -0
- sagemaker_hyperpod-3.0.0.dist-info/METADATA +534 -0
- sagemaker_hyperpod-3.0.0.dist-info/RECORD +66 -0
- sagemaker_hyperpod-3.0.0.dist-info/WHEEL +5 -0
- sagemaker_hyperpod-3.0.0.dist-info/entry_points.txt +2 -0
- sagemaker_hyperpod-3.0.0.dist-info/licenses/LICENSE +175 -0
- sagemaker_hyperpod-3.0.0.dist-info/licenses/NOTICE +1 -0
- sagemaker_hyperpod-3.0.0.dist-info/top_level.txt +1 -0
- sagemaker_hyperpod-3.0.0.dist-info/zip-safe +1 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from pydantic import ConfigDict, Field
|
|
2
|
+
from sagemaker.hyperpod.training.config.hyperpod_pytorch_job_config import (
|
|
3
|
+
_HyperPodPytorchJob,
|
|
4
|
+
)
|
|
5
|
+
from sagemaker.hyperpod.training.config.hyperpod_pytorch_job_status import (
|
|
6
|
+
HyperPodPytorchJobStatus,
|
|
7
|
+
)
|
|
8
|
+
from sagemaker.hyperpod.common.config.metadata import Metadata
|
|
9
|
+
from kubernetes import client, config
|
|
10
|
+
from typing import List, Optional, ClassVar
|
|
11
|
+
from sagemaker.hyperpod.common.utils import (
|
|
12
|
+
handle_exception,
|
|
13
|
+
get_default_namespace,
|
|
14
|
+
setup_logging,
|
|
15
|
+
)
|
|
16
|
+
import yaml
|
|
17
|
+
import logging
|
|
18
|
+
|
|
19
|
+
TRAINING_GROUP = "sagemaker.amazonaws.com"
|
|
20
|
+
API_VERSION = "v1"
|
|
21
|
+
PLURAL = "hyperpodpytorchjobs"
|
|
22
|
+
KIND = "HyperPodPyTorchJob"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class HyperPodPytorchJob(_HyperPodPytorchJob):
|
|
26
|
+
is_kubeconfig_loaded: ClassVar[bool] = False
|
|
27
|
+
|
|
28
|
+
model_config = ConfigDict(extra="forbid")
|
|
29
|
+
|
|
30
|
+
metadata: Metadata = Field(
|
|
31
|
+
description="The metadata of the HyperPodPytorchJob",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
status: Optional[HyperPodPytorchJobStatus] = Field(
|
|
35
|
+
default=None, description="The status of the HyperPodPytorchJob"
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def verify_kube_config(cls):
|
|
40
|
+
if not cls.is_kubeconfig_loaded:
|
|
41
|
+
config.load_kube_config()
|
|
42
|
+
cls.is_kubeconfig_loaded = True
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def get_logger(cls):
|
|
46
|
+
return logging.getLogger(__name__)
|
|
47
|
+
|
|
48
|
+
def create(self, debug=False):
|
|
49
|
+
self.verify_kube_config()
|
|
50
|
+
|
|
51
|
+
logger = self.get_logger()
|
|
52
|
+
logger = setup_logging(logger, debug)
|
|
53
|
+
|
|
54
|
+
spec = _HyperPodPytorchJob(**self.model_dump(by_alias=True, exclude_none=True))
|
|
55
|
+
|
|
56
|
+
if not self.metadata.namespace:
|
|
57
|
+
self.metadata.namespace = get_default_namespace()
|
|
58
|
+
|
|
59
|
+
config = {
|
|
60
|
+
"apiVersion": f"{TRAINING_GROUP}/{API_VERSION}",
|
|
61
|
+
"kind": KIND,
|
|
62
|
+
"metadata": self.metadata.model_dump(exclude_none=True),
|
|
63
|
+
"spec": spec.model_dump(exclude_none=True),
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
custom_api = client.CustomObjectsApi()
|
|
67
|
+
logger.debug(
|
|
68
|
+
"Deploying HyperPodPytorchJob with config:\n%s",
|
|
69
|
+
yaml.dump(config),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
custom_api.create_namespaced_custom_object(
|
|
74
|
+
group=TRAINING_GROUP,
|
|
75
|
+
version=API_VERSION,
|
|
76
|
+
namespace=self.metadata.namespace,
|
|
77
|
+
plural=PLURAL,
|
|
78
|
+
body=config,
|
|
79
|
+
)
|
|
80
|
+
logger.info("Successfully submitted HyperPodPytorchJob!")
|
|
81
|
+
except Exception as e:
|
|
82
|
+
logger.error(f"Failed to create HyperPodPytorchJob {self.metadata.name}!")
|
|
83
|
+
handle_exception(e, self.metadata.name, self.metadata.namespace)
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def list(cls, namespace=None) -> List["HyperPodPytorchJob"]:
|
|
87
|
+
cls.verify_kube_config()
|
|
88
|
+
|
|
89
|
+
if namespace is None:
|
|
90
|
+
namespace = get_default_namespace()
|
|
91
|
+
|
|
92
|
+
logger = cls.get_logger()
|
|
93
|
+
logger = setup_logging(logger)
|
|
94
|
+
|
|
95
|
+
custom_api = client.CustomObjectsApi()
|
|
96
|
+
|
|
97
|
+
try:
|
|
98
|
+
hp_job_list = custom_api.list_namespaced_custom_object(
|
|
99
|
+
group=TRAINING_GROUP,
|
|
100
|
+
version=API_VERSION,
|
|
101
|
+
namespace=namespace,
|
|
102
|
+
plural=PLURAL,
|
|
103
|
+
)
|
|
104
|
+
return _load_hp_job_list(hp_job_list)
|
|
105
|
+
except Exception as e:
|
|
106
|
+
logger.error(f"Failed to list HyperpodPytorchJobs!")
|
|
107
|
+
handle_exception(e, "", namespace)
|
|
108
|
+
|
|
109
|
+
def delete(self):
|
|
110
|
+
self.verify_kube_config()
|
|
111
|
+
|
|
112
|
+
logger = self.get_logger()
|
|
113
|
+
logger = setup_logging(logger)
|
|
114
|
+
|
|
115
|
+
custom_api = client.CustomObjectsApi()
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
custom_api.delete_namespaced_custom_object(
|
|
119
|
+
group=TRAINING_GROUP,
|
|
120
|
+
version=API_VERSION,
|
|
121
|
+
namespace=self.metadata.namespace,
|
|
122
|
+
plural=PLURAL,
|
|
123
|
+
name=self.metadata.name,
|
|
124
|
+
)
|
|
125
|
+
logger.info(f"Successful deleted HyperPodPytorchJob!")
|
|
126
|
+
except Exception as e:
|
|
127
|
+
logger.error(f"Failed to delete HyperPodPytorchJob {self.metadata.name}!")
|
|
128
|
+
handle_exception(e, self.metadata.name, self.metadata.namespace)
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def get(cls, name, namespace=None) -> "HyperPodPytorchJob":
|
|
132
|
+
cls.verify_kube_config()
|
|
133
|
+
|
|
134
|
+
if namespace is None:
|
|
135
|
+
namespace = get_default_namespace()
|
|
136
|
+
|
|
137
|
+
logger = cls.get_logger()
|
|
138
|
+
logger = setup_logging(logger)
|
|
139
|
+
|
|
140
|
+
custom_api = client.CustomObjectsApi()
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
response = custom_api.get_namespaced_custom_object(
|
|
144
|
+
group=TRAINING_GROUP,
|
|
145
|
+
version=API_VERSION,
|
|
146
|
+
namespace=namespace,
|
|
147
|
+
plural=PLURAL,
|
|
148
|
+
name=name,
|
|
149
|
+
)
|
|
150
|
+
return _load_hp_job(response)
|
|
151
|
+
except Exception as e:
|
|
152
|
+
logger.error(f"Failed to describe HyperPodPytorchJob {name}: {e}")
|
|
153
|
+
handle_exception(e, name, namespace)
|
|
154
|
+
|
|
155
|
+
def refresh(self) -> "HyperPodPytorchJob":
|
|
156
|
+
self.verify_kube_config()
|
|
157
|
+
|
|
158
|
+
logger = self.get_logger()
|
|
159
|
+
logger = setup_logging(logger)
|
|
160
|
+
|
|
161
|
+
custom_api = client.CustomObjectsApi()
|
|
162
|
+
|
|
163
|
+
try:
|
|
164
|
+
response = custom_api.get_namespaced_custom_object(
|
|
165
|
+
group=TRAINING_GROUP,
|
|
166
|
+
version=API_VERSION,
|
|
167
|
+
namespace=self.metadata.namespace,
|
|
168
|
+
plural=PLURAL,
|
|
169
|
+
name=self.metadata.name,
|
|
170
|
+
)
|
|
171
|
+
self.status = HyperPodPytorchJobStatus.model_validate(
|
|
172
|
+
response["status"], by_name=True
|
|
173
|
+
)
|
|
174
|
+
except Exception as e:
|
|
175
|
+
logger.error(f"Failed to refresh HyperPodPytorchJob {self.metadata.name}!")
|
|
176
|
+
handle_exception(e, self.metadata.name, self.metadata.namespace)
|
|
177
|
+
|
|
178
|
+
def list_pods(self) -> List[str]:
|
|
179
|
+
self.verify_kube_config()
|
|
180
|
+
|
|
181
|
+
logger = self.get_logger()
|
|
182
|
+
logger = setup_logging(logger)
|
|
183
|
+
|
|
184
|
+
try:
|
|
185
|
+
config.load_kube_config()
|
|
186
|
+
v1 = client.CoreV1Api()
|
|
187
|
+
|
|
188
|
+
response = v1.list_namespaced_pod(self.metadata.namespace)
|
|
189
|
+
pods = []
|
|
190
|
+
|
|
191
|
+
for pod in response.items:
|
|
192
|
+
if pod.metadata.name.startswith(f"{self.metadata.name}-pod"):
|
|
193
|
+
pods.append(pod.metadata.name)
|
|
194
|
+
return pods
|
|
195
|
+
except Exception as e:
|
|
196
|
+
logger.error(f"Failed to list pod in namespace {self.metadata.namespace}!")
|
|
197
|
+
handle_exception(e, self.metadata.name, self.metadata.namespace)
|
|
198
|
+
|
|
199
|
+
def get_logs_from_pod(self, pod_name: str, container: Optional[str] = None) -> str:
|
|
200
|
+
self.verify_kube_config()
|
|
201
|
+
|
|
202
|
+
logger = self.get_logger()
|
|
203
|
+
logger = setup_logging(logger)
|
|
204
|
+
|
|
205
|
+
if container is None:
|
|
206
|
+
# If container name is not set, get logs from the first container in the pod
|
|
207
|
+
container = self.replicaSpecs[0].template.spec.containers[0].name
|
|
208
|
+
|
|
209
|
+
try:
|
|
210
|
+
config.load_kube_config()
|
|
211
|
+
v1 = client.CoreV1Api()
|
|
212
|
+
|
|
213
|
+
logs = v1.read_namespaced_pod_log(
|
|
214
|
+
name=pod_name,
|
|
215
|
+
namespace=self.metadata.namespace,
|
|
216
|
+
timestamps=True,
|
|
217
|
+
container=container,
|
|
218
|
+
)
|
|
219
|
+
return logs
|
|
220
|
+
except Exception as e:
|
|
221
|
+
logger.error(f"Failed to get logs from pod {pod_name}!")
|
|
222
|
+
handle_exception(e, self.metadata.name, self.metadata.namespace)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _load_hp_job(response: dict) -> HyperPodPytorchJob:
|
|
226
|
+
name = response["metadata"]["name"]
|
|
227
|
+
namespace = response["metadata"]["namespace"]
|
|
228
|
+
|
|
229
|
+
spec = _HyperPodPytorchJob.model_validate(response["spec"], by_name=True)
|
|
230
|
+
metadata = Metadata(name=name, namespace=namespace)
|
|
231
|
+
|
|
232
|
+
if "status" in response:
|
|
233
|
+
status = HyperPodPytorchJobStatus.model_validate(
|
|
234
|
+
response["status"], by_name=True
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
else:
|
|
238
|
+
status = None
|
|
239
|
+
|
|
240
|
+
job = HyperPodPytorchJob(
|
|
241
|
+
metadata=metadata,
|
|
242
|
+
status=status,
|
|
243
|
+
**spec.model_dump(by_alias=True, exclude_none=True),
|
|
244
|
+
)
|
|
245
|
+
return job
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _load_hp_job_list(response: dict) -> List[HyperPodPytorchJob]:
|
|
249
|
+
job_list = []
|
|
250
|
+
for hp_job in response["items"]:
|
|
251
|
+
job = _load_hp_job(hp_job)
|
|
252
|
+
job_list.append(job)
|
|
253
|
+
return job_list
|