zenml-nightly 0.70.0.dev20241120__py3-none-any.whl → 0.70.0.dev20241125__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/artifacts/artifact_config.py +32 -4
- zenml/artifacts/utils.py +12 -24
- zenml/cli/base.py +1 -1
- zenml/client.py +4 -19
- zenml/constants.py +1 -0
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +8 -7
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +52 -1
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -1
- zenml/integrations/kubernetes/orchestrators/manifest_utils.py +3 -3
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +2 -1
- zenml/model/utils.py +0 -24
- zenml/models/__init__.py +6 -1
- zenml/models/v2/core/artifact_version.py +25 -2
- zenml/models/v2/core/model_version.py +0 -4
- zenml/models/v2/core/model_version_artifact.py +19 -76
- zenml/models/v2/core/model_version_pipeline_run.py +6 -39
- zenml/models/v2/core/service_connector.py +4 -0
- zenml/models/v2/misc/server_models.py +23 -0
- zenml/orchestrators/step_launcher.py +0 -1
- zenml/orchestrators/step_run_utils.py +4 -17
- zenml/orchestrators/step_runner.py +3 -1
- zenml/zen_server/deploy/helm/templates/_environment.tpl +117 -0
- zenml/zen_server/deploy/helm/templates/server-db-job.yaml +3 -14
- zenml/zen_server/deploy/helm/templates/server-deployment.yaml +16 -4
- zenml/zen_server/deploy/helm/templates/server-secret.yaml +2 -17
- zenml/zen_server/routers/model_versions_endpoints.py +59 -0
- zenml/zen_server/routers/server_endpoints.py +47 -0
- zenml/zen_server/routers/workspaces_endpoints.py +0 -130
- zenml/zen_server/zen_server_api.py +45 -6
- zenml/zen_stores/base_zen_store.py +2 -1
- zenml/zen_stores/migrations/utils.py +40 -24
- zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +118 -0
- zenml/zen_stores/rest_zen_store.py +42 -5
- zenml/zen_stores/schemas/model_schemas.py +10 -94
- zenml/zen_stores/schemas/user_schemas.py +0 -8
- zenml/zen_stores/schemas/workspace_schemas.py +0 -14
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/METADATA +1 -1
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/RECORD +42 -41
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/entry_points.txt +0 -0
zenml/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.70.0.
|
1
|
+
0.70.0.dev20241125
|
@@ -17,6 +17,7 @@ from typing import Any, Dict, List, Optional, Union
|
|
17
17
|
|
18
18
|
from pydantic import BaseModel, Field, model_validator
|
19
19
|
|
20
|
+
from zenml.enums import ArtifactType
|
20
21
|
from zenml.logger import get_logger
|
21
22
|
from zenml.metadata.metadata_types import MetadataType
|
22
23
|
from zenml.utils.pydantic_utils import before_validator_handler
|
@@ -36,6 +37,7 @@ class ArtifactConfig(BaseModel):
|
|
36
37
|
int, ArtifactConfig(
|
37
38
|
name="my_artifact", # override the default artifact name
|
38
39
|
version=42, # set a custom version
|
40
|
+
artifact_type=ArtifactType.MODEL, # Specify the artifact type
|
39
41
|
tags=["tag1", "tag2"], # set custom tags
|
40
42
|
)
|
41
43
|
]:
|
@@ -47,8 +49,9 @@ class ArtifactConfig(BaseModel):
|
|
47
49
|
version: The version of the artifact.
|
48
50
|
tags: The tags of the artifact.
|
49
51
|
run_metadata: Metadata to add to the artifact.
|
50
|
-
|
51
|
-
|
52
|
+
artifact_type: Optional type of the artifact. If not given, the type
|
53
|
+
specified by the materializer that is used to save this artifact
|
54
|
+
is used.
|
52
55
|
"""
|
53
56
|
|
54
57
|
name: Optional[str] = None
|
@@ -58,8 +61,7 @@ class ArtifactConfig(BaseModel):
|
|
58
61
|
tags: Optional[List[str]] = None
|
59
62
|
run_metadata: Optional[Dict[str, MetadataType]] = None
|
60
63
|
|
61
|
-
|
62
|
-
is_deployment_artifact: bool = False
|
64
|
+
artifact_type: Optional[ArtifactType] = None
|
63
65
|
|
64
66
|
@model_validator(mode="before")
|
65
67
|
@classmethod
|
@@ -70,6 +72,10 @@ class ArtifactConfig(BaseModel):
|
|
70
72
|
Args:
|
71
73
|
data: The model data.
|
72
74
|
|
75
|
+
Raises:
|
76
|
+
ValueError: If the artifact is configured to be
|
77
|
+
both a model and a deployment artifact.
|
78
|
+
|
73
79
|
Returns:
|
74
80
|
Model data without the removed attributes.
|
75
81
|
"""
|
@@ -82,4 +88,26 @@ class ArtifactConfig(BaseModel):
|
|
82
88
|
"artifact is not supported anymore."
|
83
89
|
)
|
84
90
|
|
91
|
+
is_model_artifact = data.pop("is_model_artifact", None)
|
92
|
+
is_deployment_artifact = data.pop("is_deployment_artifact", None)
|
93
|
+
|
94
|
+
if is_model_artifact and is_deployment_artifact:
|
95
|
+
raise ValueError(
|
96
|
+
"An artifact can only be a model artifact or deployment "
|
97
|
+
"artifact."
|
98
|
+
)
|
99
|
+
elif is_model_artifact:
|
100
|
+
logger.warning(
|
101
|
+
"`ArtifactConfig.is_model_artifact` is deprecated and will be "
|
102
|
+
"removed soon. Use `ArtifactConfig.artifact_type` instead."
|
103
|
+
)
|
104
|
+
data.setdefault("artifact_type", ArtifactType.MODEL)
|
105
|
+
elif is_deployment_artifact:
|
106
|
+
logger.warning(
|
107
|
+
"`ArtifactConfig.is_deployment_artifact` is deprecated and "
|
108
|
+
"will be removed soon. Use `ArtifactConfig.artifact_type` "
|
109
|
+
"instead."
|
110
|
+
)
|
111
|
+
data.setdefault("artifact_type", ArtifactType.SERVICE)
|
112
|
+
|
85
113
|
return data
|
zenml/artifacts/utils.py
CHANGED
@@ -30,7 +30,6 @@ from typing import (
|
|
30
30
|
)
|
31
31
|
from uuid import UUID, uuid4
|
32
32
|
|
33
|
-
from zenml.artifacts.artifact_config import ArtifactConfig
|
34
33
|
from zenml.artifacts.preexisting_data_materializer import (
|
35
34
|
PreexistingDataMaterializer,
|
36
35
|
)
|
@@ -118,6 +117,7 @@ def _store_artifact_data_and_prepare_request(
|
|
118
117
|
materializer_class: Type["BaseMaterializer"],
|
119
118
|
save_type: ArtifactSaveType,
|
120
119
|
version: Optional[Union[int, str]] = None,
|
120
|
+
artifact_type: Optional[ArtifactType] = None,
|
121
121
|
tags: Optional[List[str]] = None,
|
122
122
|
store_metadata: bool = True,
|
123
123
|
store_visualizations: bool = True,
|
@@ -134,6 +134,8 @@ def _store_artifact_data_and_prepare_request(
|
|
134
134
|
artifact data.
|
135
135
|
save_type: Save type of the artifact version.
|
136
136
|
version: The artifact version.
|
137
|
+
artifact_type: The artifact type. If not given, the type will be defined
|
138
|
+
by the materializer that is used to save the artifact.
|
137
139
|
tags: Tags for the artifact version.
|
138
140
|
store_metadata: Whether to store metadata for the artifact version.
|
139
141
|
store_visualizations: Whether to store visualizations for the artifact
|
@@ -176,7 +178,7 @@ def _store_artifact_data_and_prepare_request(
|
|
176
178
|
artifact_name=name,
|
177
179
|
version=version,
|
178
180
|
tags=tags,
|
179
|
-
type=materializer.ASSOCIATED_ARTIFACT_TYPE,
|
181
|
+
type=artifact_type or materializer.ASSOCIATED_ARTIFACT_TYPE,
|
180
182
|
uri=materializer.uri,
|
181
183
|
materializer=source_utils.resolve(materializer.__class__),
|
182
184
|
data_type=source_utils.resolve(data_type),
|
@@ -198,14 +200,13 @@ def save_artifact(
|
|
198
200
|
data: Any,
|
199
201
|
name: str,
|
200
202
|
version: Optional[Union[int, str]] = None,
|
203
|
+
artifact_type: Optional[ArtifactType] = None,
|
201
204
|
tags: Optional[List[str]] = None,
|
202
205
|
extract_metadata: bool = True,
|
203
206
|
include_visualizations: bool = True,
|
204
207
|
user_metadata: Optional[Dict[str, "MetadataType"]] = None,
|
205
208
|
materializer: Optional["MaterializerClassOrSource"] = None,
|
206
209
|
uri: Optional[str] = None,
|
207
|
-
is_model_artifact: bool = False,
|
208
|
-
is_deployment_artifact: bool = False,
|
209
210
|
# TODO: remove these once external artifact does not use this function anymore
|
210
211
|
save_type: ArtifactSaveType = ArtifactSaveType.MANUAL,
|
211
212
|
has_custom_name: bool = True,
|
@@ -218,6 +219,8 @@ def save_artifact(
|
|
218
219
|
version: The version of the artifact. If not provided, a new
|
219
220
|
auto-incremented version will be used.
|
220
221
|
tags: Tags to associate with the artifact.
|
222
|
+
artifact_type: The artifact type. If not given, the type will be defined
|
223
|
+
by the materializer that is used to save the artifact.
|
221
224
|
extract_metadata: If artifact metadata should be extracted and returned.
|
222
225
|
include_visualizations: If artifact visualizations should be generated.
|
223
226
|
user_metadata: User-provided metadata to store with the artifact.
|
@@ -226,8 +229,6 @@ def save_artifact(
|
|
226
229
|
uri: The URI within the artifact store to upload the artifact
|
227
230
|
to. If not provided, the artifact will be uploaded to
|
228
231
|
`custom_artifacts/{name}/{version}`.
|
229
|
-
is_model_artifact: If the artifact is a model artifact.
|
230
|
-
is_deployment_artifact: If the artifact is a deployment artifact.
|
231
232
|
save_type: The type of save operation that created the artifact version.
|
232
233
|
has_custom_name: If the artifact name is custom and should be listed in
|
233
234
|
the dashboard "Artifacts" tab.
|
@@ -273,6 +274,7 @@ def save_artifact(
|
|
273
274
|
materializer_class=materializer_class,
|
274
275
|
save_type=save_type,
|
275
276
|
version=version,
|
277
|
+
artifact_type=artifact_type,
|
276
278
|
tags=tags,
|
277
279
|
store_metadata=extract_metadata,
|
278
280
|
store_visualizations=include_visualizations,
|
@@ -286,8 +288,6 @@ def save_artifact(
|
|
286
288
|
if save_type == ArtifactSaveType.MANUAL:
|
287
289
|
_link_artifact_version_to_the_step_and_model(
|
288
290
|
artifact_version=artifact_version,
|
289
|
-
is_model_artifact=is_model_artifact,
|
290
|
-
is_deployment_artifact=is_deployment_artifact,
|
291
291
|
)
|
292
292
|
|
293
293
|
return artifact_version
|
@@ -297,10 +297,9 @@ def register_artifact(
|
|
297
297
|
folder_or_file_uri: str,
|
298
298
|
name: str,
|
299
299
|
version: Optional[Union[int, str]] = None,
|
300
|
+
artifact_type: Optional[ArtifactType] = None,
|
300
301
|
tags: Optional[List[str]] = None,
|
301
302
|
has_custom_name: bool = True,
|
302
|
-
is_model_artifact: bool = False,
|
303
|
-
is_deployment_artifact: bool = False,
|
304
303
|
artifact_metadata: Dict[str, "MetadataType"] = {},
|
305
304
|
) -> "ArtifactVersionResponse":
|
306
305
|
"""Register existing data stored in the artifact store as a ZenML Artifact.
|
@@ -311,11 +310,11 @@ def register_artifact(
|
|
311
310
|
name: The name of the artifact.
|
312
311
|
version: The version of the artifact. If not provided, a new
|
313
312
|
auto-incremented version will be used.
|
313
|
+
artifact_type: The artifact type. If not given, the type will default
|
314
|
+
to `data`.
|
314
315
|
tags: Tags to associate with the artifact.
|
315
316
|
has_custom_name: If the artifact name is custom and should be listed in
|
316
317
|
the dashboard "Artifacts" tab.
|
317
|
-
is_model_artifact: If the artifact is a model artifact.
|
318
|
-
is_deployment_artifact: If the artifact is a deployment artifact.
|
319
318
|
artifact_metadata: Metadata dictionary to attach to the artifact version.
|
320
319
|
|
321
320
|
Returns:
|
@@ -346,7 +345,7 @@ def register_artifact(
|
|
346
345
|
artifact_name=name,
|
347
346
|
version=version,
|
348
347
|
tags=tags,
|
349
|
-
type=ArtifactType.DATA,
|
348
|
+
type=artifact_type or ArtifactType.DATA,
|
350
349
|
save_type=ArtifactSaveType.PREEXISTING,
|
351
350
|
uri=folder_or_file_uri,
|
352
351
|
materializer=source_utils.resolve(PreexistingDataMaterializer),
|
@@ -365,8 +364,6 @@ def register_artifact(
|
|
365
364
|
|
366
365
|
_link_artifact_version_to_the_step_and_model(
|
367
366
|
artifact_version=artifact_version,
|
368
|
-
is_model_artifact=is_model_artifact,
|
369
|
-
is_deployment_artifact=is_deployment_artifact,
|
370
367
|
)
|
371
368
|
|
372
369
|
return artifact_version
|
@@ -674,8 +671,6 @@ def _check_if_artifact_with_given_uri_already_registered(
|
|
674
671
|
|
675
672
|
def _link_artifact_version_to_the_step_and_model(
|
676
673
|
artifact_version: ArtifactVersionResponse,
|
677
|
-
is_model_artifact: bool,
|
678
|
-
is_deployment_artifact: bool,
|
679
674
|
) -> None:
|
680
675
|
"""Link an artifact version to the step and its' context model.
|
681
676
|
|
@@ -685,8 +680,6 @@ def _link_artifact_version_to_the_step_and_model(
|
|
685
680
|
|
686
681
|
Args:
|
687
682
|
artifact_version: The artifact version to link.
|
688
|
-
is_model_artifact: Whether the artifact is a model artifact.
|
689
|
-
is_deployment_artifact: Whether the artifact is a deployment artifact.
|
690
683
|
"""
|
691
684
|
client = Client()
|
692
685
|
try:
|
@@ -706,14 +699,9 @@ def _link_artifact_version_to_the_step_and_model(
|
|
706
699
|
link_artifact_version_to_model_version,
|
707
700
|
)
|
708
701
|
|
709
|
-
artifact_config = ArtifactConfig(
|
710
|
-
is_model_artifact=is_model_artifact,
|
711
|
-
is_deployment_artifact=is_deployment_artifact,
|
712
|
-
)
|
713
702
|
link_artifact_version_to_model_version(
|
714
703
|
artifact_version=artifact_version,
|
715
704
|
model_version=step_context.model_version,
|
716
|
-
artifact_config=artifact_config,
|
717
705
|
)
|
718
706
|
except (RuntimeError, StepContextError):
|
719
707
|
logger.debug(f"Unable to link saved artifact to {error_message}.")
|
zenml/cli/base.py
CHANGED
@@ -79,7 +79,7 @@ class ZenMLProjectTemplateLocation(BaseModel):
|
|
79
79
|
ZENML_PROJECT_TEMPLATES = dict(
|
80
80
|
e2e_batch=ZenMLProjectTemplateLocation(
|
81
81
|
github_url="zenml-io/template-e2e-batch",
|
82
|
-
github_tag="2024.
|
82
|
+
github_tag="2024.11.20", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
83
83
|
),
|
84
84
|
starter=ZenMLProjectTemplateLocation(
|
85
85
|
github_url="zenml-io/template-starter",
|
zenml/client.py
CHANGED
@@ -4220,6 +4220,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4220
4220
|
materializer: Optional[str] = None,
|
4221
4221
|
workspace_id: Optional[Union[str, UUID]] = None,
|
4222
4222
|
user_id: Optional[Union[str, UUID]] = None,
|
4223
|
+
model_version_id: Optional[Union[str, UUID]] = None,
|
4223
4224
|
only_unused: Optional[bool] = False,
|
4224
4225
|
has_custom_name: Optional[bool] = None,
|
4225
4226
|
user: Optional[Union[UUID, str]] = None,
|
@@ -4249,7 +4250,8 @@ class Client(metaclass=ClientMetaClass):
|
|
4249
4250
|
uri: The uri of the artifact to filter by.
|
4250
4251
|
materializer: The materializer of the artifact to filter by.
|
4251
4252
|
workspace_id: The id of the workspace to filter by.
|
4252
|
-
user_id: The
|
4253
|
+
user_id: The id of the user to filter by.
|
4254
|
+
model_version_id: Filter by model version ID.
|
4253
4255
|
only_unused: Only return artifact versions that are not used in
|
4254
4256
|
any pipeline runs.
|
4255
4257
|
has_custom_name: Filter artifacts with/without custom names.
|
@@ -4283,6 +4285,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4283
4285
|
materializer=materializer,
|
4284
4286
|
workspace_id=workspace_id,
|
4285
4287
|
user_id=user_id,
|
4288
|
+
model_version_id=model_version_id,
|
4286
4289
|
only_unused=only_unused,
|
4287
4290
|
has_custom_name=has_custom_name,
|
4288
4291
|
tag=tag,
|
@@ -6435,9 +6438,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6435
6438
|
logical_operator: LogicalOperators = LogicalOperators.AND,
|
6436
6439
|
created: Optional[Union[datetime, str]] = None,
|
6437
6440
|
updated: Optional[Union[datetime, str]] = None,
|
6438
|
-
workspace_id: Optional[Union[UUID, str]] = None,
|
6439
|
-
user_id: Optional[Union[UUID, str]] = None,
|
6440
|
-
model_id: Optional[Union[UUID, str]] = None,
|
6441
6441
|
model_version_id: Optional[Union[UUID, str]] = None,
|
6442
6442
|
artifact_version_id: Optional[Union[UUID, str]] = None,
|
6443
6443
|
artifact_name: Optional[str] = None,
|
@@ -6457,9 +6457,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6457
6457
|
logical_operator: Which logical operator to use [and, or]
|
6458
6458
|
created: Use to filter by time of creation
|
6459
6459
|
updated: Use the last updated date for filtering
|
6460
|
-
workspace_id: Use the workspace id for filtering
|
6461
|
-
user_id: Use the user id for filtering
|
6462
|
-
model_id: Use the model id for filtering
|
6463
6460
|
model_version_id: Use the model version id for filtering
|
6464
6461
|
artifact_version_id: Use the artifact id for filtering
|
6465
6462
|
artifact_name: Use the artifact name for filtering
|
@@ -6482,9 +6479,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6482
6479
|
size=size,
|
6483
6480
|
created=created,
|
6484
6481
|
updated=updated,
|
6485
|
-
workspace_id=workspace_id,
|
6486
|
-
user_id=user_id,
|
6487
|
-
model_id=model_id,
|
6488
6482
|
model_version_id=model_version_id,
|
6489
6483
|
artifact_version_id=artifact_version_id,
|
6490
6484
|
artifact_name=artifact_name,
|
@@ -6556,9 +6550,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6556
6550
|
logical_operator: LogicalOperators = LogicalOperators.AND,
|
6557
6551
|
created: Optional[Union[datetime, str]] = None,
|
6558
6552
|
updated: Optional[Union[datetime, str]] = None,
|
6559
|
-
workspace_id: Optional[Union[UUID, str]] = None,
|
6560
|
-
user_id: Optional[Union[UUID, str]] = None,
|
6561
|
-
model_id: Optional[Union[UUID, str]] = None,
|
6562
6553
|
model_version_id: Optional[Union[UUID, str]] = None,
|
6563
6554
|
pipeline_run_id: Optional[Union[UUID, str]] = None,
|
6564
6555
|
pipeline_run_name: Optional[str] = None,
|
@@ -6574,9 +6565,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6574
6565
|
logical_operator: Which logical operator to use [and, or]
|
6575
6566
|
created: Use to filter by time of creation
|
6576
6567
|
updated: Use the last updated date for filtering
|
6577
|
-
workspace_id: Use the workspace id for filtering
|
6578
|
-
user_id: Use the user id for filtering
|
6579
|
-
model_id: Use the model id for filtering
|
6580
6568
|
model_version_id: Use the model version id for filtering
|
6581
6569
|
pipeline_run_id: Use the pipeline run id for filtering
|
6582
6570
|
pipeline_run_name: Use the pipeline run name for filtering
|
@@ -6595,9 +6583,6 @@ class Client(metaclass=ClientMetaClass):
|
|
6595
6583
|
size=size,
|
6596
6584
|
created=created,
|
6597
6585
|
updated=updated,
|
6598
|
-
workspace_id=workspace_id,
|
6599
|
-
user_id=user_id,
|
6600
|
-
model_id=model_id,
|
6601
6586
|
model_version_id=model_version_id,
|
6602
6587
|
pipeline_run_id=pipeline_run_id,
|
6603
6588
|
pipeline_run_name=pipeline_run_name,
|
zenml/constants.py
CHANGED
@@ -42,8 +42,8 @@ from kubernetes import config as k8s_config
|
|
42
42
|
from kubernetes.client.rest import ApiException
|
43
43
|
|
44
44
|
from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
|
45
|
-
build_cluster_role_binding_manifest_for_service_account,
|
46
45
|
build_namespace_manifest,
|
46
|
+
build_role_binding_manifest_for_service_account,
|
47
47
|
build_service_account_manifest,
|
48
48
|
)
|
49
49
|
from zenml.logger import get_logger
|
@@ -288,7 +288,7 @@ def create_edit_service_account(
|
|
288
288
|
rbac_api: k8s_client.RbacAuthorizationV1Api,
|
289
289
|
service_account_name: str,
|
290
290
|
namespace: str,
|
291
|
-
|
291
|
+
role_binding_name: str = "zenml-edit",
|
292
292
|
) -> None:
|
293
293
|
"""Create a new Kubernetes service account with "edit" rights.
|
294
294
|
|
@@ -297,16 +297,17 @@ def create_edit_service_account(
|
|
297
297
|
rbac_api: Client of Rbac Authorization V1 API of Kubernetes API.
|
298
298
|
service_account_name: Name of the service account.
|
299
299
|
namespace: Kubernetes namespace. Defaults to "default".
|
300
|
-
|
301
|
-
Defaults to "zenml-edit".
|
300
|
+
role_binding_name: Name of the role binding. Defaults to "zenml-edit".
|
302
301
|
"""
|
303
|
-
|
304
|
-
name=
|
302
|
+
rb_manifest = build_role_binding_manifest_for_service_account(
|
303
|
+
name=role_binding_name,
|
305
304
|
role_name="edit",
|
306
305
|
service_account_name=service_account_name,
|
307
306
|
namespace=namespace,
|
308
307
|
)
|
309
|
-
_if_not_exists(rbac_api.
|
308
|
+
_if_not_exists(rbac_api.create_namespaced_role_binding)(
|
309
|
+
namespace=namespace, body=rb_manifest
|
310
|
+
)
|
310
311
|
|
311
312
|
sa_manifest = build_service_account_manifest(
|
312
313
|
name=service_account_name, namespace=namespace
|
@@ -59,6 +59,7 @@ from zenml.integrations.kubernetes.orchestrators.manifest_utils import (
|
|
59
59
|
build_cron_job_manifest,
|
60
60
|
build_pod_manifest,
|
61
61
|
)
|
62
|
+
from zenml.integrations.kubernetes.pod_settings import KubernetesPodSettings
|
62
63
|
from zenml.logger import get_logger
|
63
64
|
from zenml.orchestrators import ContainerizedOrchestrator
|
64
65
|
from zenml.orchestrators.utils import get_orchestrator_run_name
|
@@ -328,6 +329,45 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
|
|
328
329
|
custom_validation_function=_validate_local_requirements,
|
329
330
|
)
|
330
331
|
|
332
|
+
@classmethod
|
333
|
+
def apply_default_resource_requests(
|
334
|
+
cls,
|
335
|
+
memory: str,
|
336
|
+
cpu: Optional[str] = None,
|
337
|
+
pod_settings: Optional[KubernetesPodSettings] = None,
|
338
|
+
) -> KubernetesPodSettings:
|
339
|
+
"""Applies default resource requests to a pod settings object.
|
340
|
+
|
341
|
+
Args:
|
342
|
+
memory: The memory resource request.
|
343
|
+
cpu: The CPU resource request.
|
344
|
+
pod_settings: The pod settings to update. A new one will be created
|
345
|
+
if not provided.
|
346
|
+
|
347
|
+
Returns:
|
348
|
+
The new or updated pod settings.
|
349
|
+
"""
|
350
|
+
resources = {
|
351
|
+
"requests": {"memory": memory},
|
352
|
+
}
|
353
|
+
if cpu:
|
354
|
+
resources["requests"]["cpu"] = cpu
|
355
|
+
if not pod_settings:
|
356
|
+
pod_settings = KubernetesPodSettings(resources=resources)
|
357
|
+
elif not pod_settings.resources:
|
358
|
+
# We can't update the pod settings in place (because it's a frozen
|
359
|
+
# pydantic model), so we have to create a new one.
|
360
|
+
pod_settings = KubernetesPodSettings(
|
361
|
+
**pod_settings.model_dump(exclude_unset=True),
|
362
|
+
resources=resources,
|
363
|
+
)
|
364
|
+
else:
|
365
|
+
set_requests = pod_settings.resources.get("requests", {})
|
366
|
+
resources["requests"].update(set_requests)
|
367
|
+
pod_settings.resources["requests"] = resources["requests"]
|
368
|
+
|
369
|
+
return pod_settings
|
370
|
+
|
331
371
|
def prepare_or_run_pipeline(
|
332
372
|
self,
|
333
373
|
deployment: "PipelineDeploymentResponse",
|
@@ -422,6 +462,17 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
|
|
422
462
|
)
|
423
463
|
return
|
424
464
|
|
465
|
+
# We set some default minimum resource requests for the orchestrator pod
|
466
|
+
# here if the user has not specified any, because the orchestrator pod
|
467
|
+
# takes up some memory resources itself and, if not specified, the pod
|
468
|
+
# will be scheduled on any node regardless of available memory and risk
|
469
|
+
# negatively impacting or even crashing the node due to memory pressure.
|
470
|
+
orchestrator_pod_settings = self.apply_default_resource_requests(
|
471
|
+
memory="400Mi",
|
472
|
+
cpu="100m",
|
473
|
+
pod_settings=settings.orchestrator_pod_settings,
|
474
|
+
)
|
475
|
+
|
425
476
|
# Create and run the orchestrator pod.
|
426
477
|
pod_manifest = build_pod_manifest(
|
427
478
|
run_name=orchestrator_run_name,
|
@@ -431,7 +482,7 @@ class KubernetesOrchestrator(ContainerizedOrchestrator):
|
|
431
482
|
command=command,
|
432
483
|
args=args,
|
433
484
|
privileged=False,
|
434
|
-
pod_settings=
|
485
|
+
pod_settings=orchestrator_pod_settings,
|
435
486
|
service_account_name=service_account_name,
|
436
487
|
env=environment,
|
437
488
|
mount_local_stores=self.config.is_local,
|
@@ -116,6 +116,16 @@ def main() -> None:
|
|
116
116
|
env = get_config_environment_vars()
|
117
117
|
env[ENV_ZENML_KUBERNETES_RUN_ID] = orchestrator_run_id
|
118
118
|
|
119
|
+
# We set some default minimum memory resource requests for the step pod
|
120
|
+
# here if the user has not specified any, because the step pod takes up
|
121
|
+
# some memory resources itself and, if not specified, the pod will be
|
122
|
+
# scheduled on any node regardless of available memory and risk
|
123
|
+
# negatively impacting or even crashing the node due to memory pressure.
|
124
|
+
pod_settings = KubernetesOrchestrator.apply_default_resource_requests(
|
125
|
+
memory="400Mi",
|
126
|
+
pod_settings=settings.pod_settings,
|
127
|
+
)
|
128
|
+
|
119
129
|
# Define Kubernetes pod manifest.
|
120
130
|
pod_manifest = build_pod_manifest(
|
121
131
|
pod_name=pod_name,
|
@@ -126,7 +136,7 @@ def main() -> None:
|
|
126
136
|
args=step_args,
|
127
137
|
env=env,
|
128
138
|
privileged=settings.privileged,
|
129
|
-
pod_settings=
|
139
|
+
pod_settings=pod_settings,
|
130
140
|
service_account_name=settings.step_pod_service_account_name
|
131
141
|
or settings.service_account_name,
|
132
142
|
mount_local_stores=mount_local_stores,
|
@@ -304,13 +304,13 @@ def build_cron_job_manifest(
|
|
304
304
|
return job_manifest
|
305
305
|
|
306
306
|
|
307
|
-
def
|
307
|
+
def build_role_binding_manifest_for_service_account(
|
308
308
|
name: str,
|
309
309
|
role_name: str,
|
310
310
|
service_account_name: str,
|
311
311
|
namespace: str = "default",
|
312
312
|
) -> Dict[str, Any]:
|
313
|
-
"""Build a manifest for a
|
313
|
+
"""Build a manifest for a role binding of a service account.
|
314
314
|
|
315
315
|
Args:
|
316
316
|
name: Name of the cluster role binding.
|
@@ -323,7 +323,7 @@ def build_cluster_role_binding_manifest_for_service_account(
|
|
323
323
|
"""
|
324
324
|
return {
|
325
325
|
"apiVersion": "rbac.authorization.k8s.io/v1",
|
326
|
-
"kind": "
|
326
|
+
"kind": "RoleBinding",
|
327
327
|
"metadata": {"name": name},
|
328
328
|
"subjects": [
|
329
329
|
{
|
@@ -26,6 +26,7 @@ from typing import Any, List, Optional
|
|
26
26
|
from kubernetes import client as k8s_client
|
27
27
|
from kubernetes import config as k8s_config
|
28
28
|
from pydantic import Field
|
29
|
+
from urllib3.exceptions import HTTPError
|
29
30
|
|
30
31
|
from zenml.constants import KUBERNETES_CLUSTER_RESOURCE_TYPE
|
31
32
|
from zenml.exceptions import AuthorizationException
|
@@ -572,7 +573,7 @@ class KubernetesServiceConnector(ServiceConnector):
|
|
572
573
|
auth_settings=["BearerToken"],
|
573
574
|
response_type="VersionInfo",
|
574
575
|
)
|
575
|
-
except k8s_client.ApiException as err:
|
576
|
+
except (k8s_client.ApiException, HTTPError) as err:
|
576
577
|
raise AuthorizationException(
|
577
578
|
f"failed to verify Kubernetes cluster access: {err}"
|
578
579
|
) from err
|
zenml/model/utils.py
CHANGED
@@ -16,7 +16,6 @@
|
|
16
16
|
from typing import Dict, Optional, Union
|
17
17
|
from uuid import UUID
|
18
18
|
|
19
|
-
from zenml.artifacts.artifact_config import ArtifactConfig
|
20
19
|
from zenml.client import Client
|
21
20
|
from zenml.enums import ModelStages
|
22
21
|
from zenml.exceptions import StepContextError
|
@@ -81,32 +80,18 @@ def log_model_metadata(
|
|
81
80
|
def link_artifact_version_to_model_version(
|
82
81
|
artifact_version: ArtifactVersionResponse,
|
83
82
|
model_version: ModelVersionResponse,
|
84
|
-
artifact_config: Optional[ArtifactConfig] = None,
|
85
83
|
) -> None:
|
86
84
|
"""Link an artifact version to a model version.
|
87
85
|
|
88
86
|
Args:
|
89
87
|
artifact_version: The artifact version to link.
|
90
88
|
model_version: The model version to link.
|
91
|
-
artifact_config: Output artifact configuration.
|
92
89
|
"""
|
93
|
-
if artifact_config:
|
94
|
-
is_model_artifact = artifact_config.is_model_artifact
|
95
|
-
is_deployment_artifact = artifact_config.is_deployment_artifact
|
96
|
-
else:
|
97
|
-
is_model_artifact = False
|
98
|
-
is_deployment_artifact = False
|
99
|
-
|
100
90
|
client = Client()
|
101
91
|
client.zen_store.create_model_version_artifact_link(
|
102
92
|
ModelVersionArtifactRequest(
|
103
|
-
user=client.active_user.id,
|
104
|
-
workspace=client.active_workspace.id,
|
105
93
|
artifact_version=artifact_version.id,
|
106
|
-
model=model_version.model.id,
|
107
94
|
model_version=model_version.id,
|
108
|
-
is_model_artifact=is_model_artifact,
|
109
|
-
is_deployment_artifact=is_deployment_artifact,
|
110
95
|
)
|
111
96
|
)
|
112
97
|
|
@@ -114,16 +99,12 @@ def link_artifact_version_to_model_version(
|
|
114
99
|
def link_artifact_to_model(
|
115
100
|
artifact_version: ArtifactVersionResponse,
|
116
101
|
model: Optional["Model"] = None,
|
117
|
-
is_model_artifact: bool = False,
|
118
|
-
is_deployment_artifact: bool = False,
|
119
102
|
) -> None:
|
120
103
|
"""Link the artifact to the model.
|
121
104
|
|
122
105
|
Args:
|
123
106
|
artifact_version: The artifact version to link.
|
124
107
|
model: The model to link to.
|
125
|
-
is_model_artifact: Whether the artifact is a model artifact.
|
126
|
-
is_deployment_artifact: Whether the artifact is a deployment artifact.
|
127
108
|
|
128
109
|
Raises:
|
129
110
|
RuntimeError: If called outside of a step.
|
@@ -145,14 +126,9 @@ def link_artifact_to_model(
|
|
145
126
|
)
|
146
127
|
|
147
128
|
model_version = model._get_or_create_model_version()
|
148
|
-
artifact_config = ArtifactConfig(
|
149
|
-
is_model_artifact=is_model_artifact,
|
150
|
-
is_deployment_artifact=is_deployment_artifact,
|
151
|
-
)
|
152
129
|
link_artifact_version_to_model_version(
|
153
130
|
artifact_version=artifact_version,
|
154
131
|
model_version=model_version,
|
155
|
-
artifact_config=artifact_config,
|
156
132
|
)
|
157
133
|
|
158
134
|
|
zenml/models/__init__.py
CHANGED
@@ -328,7 +328,11 @@ from zenml.models.v2.misc.service_connector_type import (
|
|
328
328
|
ServiceConnectorTypedResourcesModel,
|
329
329
|
ResourceTypeModel,
|
330
330
|
)
|
331
|
-
from zenml.models.v2.misc.server_models import
|
331
|
+
from zenml.models.v2.misc.server_models import (
|
332
|
+
ServerDatabaseType,
|
333
|
+
ServerLoadInfo,
|
334
|
+
ServerModel,
|
335
|
+
)
|
332
336
|
from zenml.models.v2.core.trigger import (
|
333
337
|
TriggerRequest,
|
334
338
|
TriggerFilter,
|
@@ -731,6 +735,7 @@ __all__ = [
|
|
731
735
|
"ExternalUserModel",
|
732
736
|
"BuildItem",
|
733
737
|
"LoadedVisualization",
|
738
|
+
"ServerLoadInfo",
|
734
739
|
"ServerModel",
|
735
740
|
"ServerDatabaseType",
|
736
741
|
"ServerDeploymentType",
|