zenml-nightly 0.70.0.dev20241120__py3-none-any.whl → 0.70.0.dev20241125__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zenml/VERSION +1 -1
- zenml/artifacts/artifact_config.py +32 -4
- zenml/artifacts/utils.py +12 -24
- zenml/cli/base.py +1 -1
- zenml/client.py +4 -19
- zenml/constants.py +1 -0
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +8 -7
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +52 -1
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -1
- zenml/integrations/kubernetes/orchestrators/manifest_utils.py +3 -3
- zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +2 -1
- zenml/model/utils.py +0 -24
- zenml/models/__init__.py +6 -1
- zenml/models/v2/core/artifact_version.py +25 -2
- zenml/models/v2/core/model_version.py +0 -4
- zenml/models/v2/core/model_version_artifact.py +19 -76
- zenml/models/v2/core/model_version_pipeline_run.py +6 -39
- zenml/models/v2/core/service_connector.py +4 -0
- zenml/models/v2/misc/server_models.py +23 -0
- zenml/orchestrators/step_launcher.py +0 -1
- zenml/orchestrators/step_run_utils.py +4 -17
- zenml/orchestrators/step_runner.py +3 -1
- zenml/zen_server/deploy/helm/templates/_environment.tpl +117 -0
- zenml/zen_server/deploy/helm/templates/server-db-job.yaml +3 -14
- zenml/zen_server/deploy/helm/templates/server-deployment.yaml +16 -4
- zenml/zen_server/deploy/helm/templates/server-secret.yaml +2 -17
- zenml/zen_server/routers/model_versions_endpoints.py +59 -0
- zenml/zen_server/routers/server_endpoints.py +47 -0
- zenml/zen_server/routers/workspaces_endpoints.py +0 -130
- zenml/zen_server/zen_server_api.py +45 -6
- zenml/zen_stores/base_zen_store.py +2 -1
- zenml/zen_stores/migrations/utils.py +40 -24
- zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +118 -0
- zenml/zen_stores/rest_zen_store.py +42 -5
- zenml/zen_stores/schemas/model_schemas.py +10 -94
- zenml/zen_stores/schemas/user_schemas.py +0 -8
- zenml/zen_stores/schemas/workspace_schemas.py +0 -14
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/METADATA +1 -1
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/RECORD +42 -41
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/entry_points.txt +0 -0
@@ -37,7 +37,7 @@ from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
|
|
37
37
|
from zenml.enums import ArtifactSaveType, ArtifactType, GenericFilterOps
|
38
38
|
from zenml.logger import get_logger
|
39
39
|
from zenml.metadata.metadata_types import MetadataType
|
40
|
-
from zenml.models.v2.base.filter import StrFilter
|
40
|
+
from zenml.models.v2.base.filter import FilterGenerator, StrFilter
|
41
41
|
from zenml.models.v2.base.scoped import (
|
42
42
|
WorkspaceScopedRequest,
|
43
43
|
WorkspaceScopedResponse,
|
@@ -474,6 +474,7 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
|
|
474
474
|
"user",
|
475
475
|
"model",
|
476
476
|
"pipeline_run",
|
477
|
+
"model_version_id",
|
477
478
|
"run_metadata",
|
478
479
|
]
|
479
480
|
artifact_id: Optional[Union[UUID, str]] = Field(
|
@@ -525,6 +526,11 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
|
|
525
526
|
description="User that produced this artifact",
|
526
527
|
union_mode="left_to_right",
|
527
528
|
)
|
529
|
+
model_version_id: Optional[Union[UUID, str]] = Field(
|
530
|
+
default=None,
|
531
|
+
description="ID of the model version that is associated with this artifact version.",
|
532
|
+
union_mode="left_to_right",
|
533
|
+
)
|
528
534
|
only_unused: Optional[bool] = Field(
|
529
535
|
default=False, description="Filter only for unused artifacts"
|
530
536
|
)
|
@@ -568,6 +574,7 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
|
|
568
574
|
ArtifactVersionSchema,
|
569
575
|
ModelSchema,
|
570
576
|
ModelVersionArtifactSchema,
|
577
|
+
ModelVersionSchema,
|
571
578
|
PipelineRunSchema,
|
572
579
|
RunMetadataSchema,
|
573
580
|
StepRunInputArtifactSchema,
|
@@ -600,6 +607,20 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
|
|
600
607
|
)
|
601
608
|
custom_filters.append(unused_filter)
|
602
609
|
|
610
|
+
if self.model_version_id:
|
611
|
+
value, operator = self._resolve_operator(self.model_version_id)
|
612
|
+
|
613
|
+
model_version_filter = and_(
|
614
|
+
ArtifactVersionSchema.id
|
615
|
+
== ModelVersionArtifactSchema.artifact_version_id,
|
616
|
+
ModelVersionArtifactSchema.model_version_id
|
617
|
+
== ModelVersionSchema.id,
|
618
|
+
FilterGenerator(ModelVersionSchema)
|
619
|
+
.define_filter(column="id", value=value, operator=operator)
|
620
|
+
.generate_query_conditions(ModelVersionSchema),
|
621
|
+
)
|
622
|
+
custom_filters.append(model_version_filter)
|
623
|
+
|
603
624
|
if self.has_custom_name is not None:
|
604
625
|
custom_name_filter = and_(
|
605
626
|
ArtifactVersionSchema.artifact_id == ArtifactSchema.id,
|
@@ -622,7 +643,9 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
|
|
622
643
|
model_filter = and_(
|
623
644
|
ArtifactVersionSchema.id
|
624
645
|
== ModelVersionArtifactSchema.artifact_version_id,
|
625
|
-
ModelVersionArtifactSchema.
|
646
|
+
ModelVersionArtifactSchema.model_version_id
|
647
|
+
== ModelVersionSchema.id,
|
648
|
+
ModelVersionSchema.model_id == ModelSchema.id,
|
626
649
|
self.generate_name_or_id_query_conditions(
|
627
650
|
value=self.model, table=ModelSchema
|
628
651
|
),
|
@@ -576,10 +576,6 @@ class ModelVersionResponse(
|
|
576
576
|
force=force,
|
577
577
|
)
|
578
578
|
|
579
|
-
# TODO in https://zenml.atlassian.net/browse/OSS-2433
|
580
|
-
# def generate_model_card(self, template_name: str) -> str:
|
581
|
-
# """Return HTML/PDF based on input template"""
|
582
|
-
|
583
579
|
|
584
580
|
# ------------------ Filter Model ------------------
|
585
581
|
|
@@ -16,20 +16,17 @@
|
|
16
16
|
from typing import TYPE_CHECKING, List, Optional, Union
|
17
17
|
from uuid import UUID
|
18
18
|
|
19
|
-
from pydantic import ConfigDict, Field
|
19
|
+
from pydantic import ConfigDict, Field
|
20
20
|
|
21
21
|
from zenml.enums import GenericFilterOps
|
22
22
|
from zenml.models.v2.base.base import (
|
23
23
|
BaseDatedResponseBody,
|
24
24
|
BaseIdentifiedResponse,
|
25
|
+
BaseRequest,
|
25
26
|
BaseResponseMetadata,
|
26
27
|
BaseResponseResources,
|
27
28
|
)
|
28
|
-
from zenml.models.v2.base.filter import StrFilter
|
29
|
-
from zenml.models.v2.base.scoped import (
|
30
|
-
WorkspaceScopedFilter,
|
31
|
-
WorkspaceScopedRequest,
|
32
|
-
)
|
29
|
+
from zenml.models.v2.base.filter import BaseFilter, StrFilter
|
33
30
|
|
34
31
|
if TYPE_CHECKING:
|
35
32
|
from sqlalchemy.sql.elements import ColumnElement
|
@@ -40,14 +37,11 @@ if TYPE_CHECKING:
|
|
40
37
|
# ------------------ Request Model ------------------
|
41
38
|
|
42
39
|
|
43
|
-
class ModelVersionArtifactRequest(
|
40
|
+
class ModelVersionArtifactRequest(BaseRequest):
|
44
41
|
"""Request model for links between model versions and artifacts."""
|
45
42
|
|
46
|
-
model: UUID
|
47
43
|
model_version: UUID
|
48
44
|
artifact_version: UUID
|
49
|
-
is_model_artifact: bool = False
|
50
|
-
is_deployment_artifact: bool = False
|
51
45
|
|
52
46
|
# TODO: In Pydantic v2, the `model_` is a protected namespaces for all
|
53
47
|
# fields defined under base models. If not handled, this raises a warning.
|
@@ -57,15 +51,6 @@ class ModelVersionArtifactRequest(WorkspaceScopedRequest):
|
|
57
51
|
# careful we might overwrite some fields protected by pydantic.
|
58
52
|
model_config = ConfigDict(protected_namespaces=())
|
59
53
|
|
60
|
-
@model_validator(mode="after")
|
61
|
-
def _validate_is_endpoint_artifact(self) -> "ModelVersionArtifactRequest":
|
62
|
-
if self.is_model_artifact and self.is_deployment_artifact:
|
63
|
-
raise ValueError(
|
64
|
-
"Artifact cannot be a model artifact and deployment artifact "
|
65
|
-
"at the same time."
|
66
|
-
)
|
67
|
-
return self
|
68
|
-
|
69
54
|
|
70
55
|
# ------------------ Update Model ------------------
|
71
56
|
|
@@ -77,11 +62,8 @@ class ModelVersionArtifactRequest(WorkspaceScopedRequest):
|
|
77
62
|
class ModelVersionArtifactResponseBody(BaseDatedResponseBody):
|
78
63
|
"""Response body for links between model versions and artifacts."""
|
79
64
|
|
80
|
-
model: UUID
|
81
65
|
model_version: UUID
|
82
66
|
artifact_version: "ArtifactVersionResponse"
|
83
|
-
is_model_artifact: bool = False
|
84
|
-
is_deployment_artifact: bool = False
|
85
67
|
|
86
68
|
# TODO: In Pydantic v2, the `model_` is a protected namespaces for all
|
87
69
|
# fields defined under base models. If not handled, this raises a warning.
|
@@ -105,16 +87,6 @@ class ModelVersionArtifactResponse(
|
|
105
87
|
):
|
106
88
|
"""Response model for links between model versions and artifacts."""
|
107
89
|
|
108
|
-
# Body and metadata properties
|
109
|
-
@property
|
110
|
-
def model(self) -> UUID:
|
111
|
-
"""The `model` property.
|
112
|
-
|
113
|
-
Returns:
|
114
|
-
the value of the property.
|
115
|
-
"""
|
116
|
-
return self.get_body().model
|
117
|
-
|
118
90
|
@property
|
119
91
|
def model_version(self) -> UUID:
|
120
92
|
"""The `model_version` property.
|
@@ -133,34 +105,16 @@ class ModelVersionArtifactResponse(
|
|
133
105
|
"""
|
134
106
|
return self.get_body().artifact_version
|
135
107
|
|
136
|
-
@property
|
137
|
-
def is_model_artifact(self) -> bool:
|
138
|
-
"""The `is_model_artifact` property.
|
139
|
-
|
140
|
-
Returns:
|
141
|
-
the value of the property.
|
142
|
-
"""
|
143
|
-
return self.get_body().is_model_artifact
|
144
|
-
|
145
|
-
@property
|
146
|
-
def is_deployment_artifact(self) -> bool:
|
147
|
-
"""The `is_deployment_artifact` property.
|
148
|
-
|
149
|
-
Returns:
|
150
|
-
the value of the property.
|
151
|
-
"""
|
152
|
-
return self.get_body().is_deployment_artifact
|
153
|
-
|
154
108
|
|
155
109
|
# ------------------ Filter Model ------------------
|
156
110
|
|
157
111
|
|
158
|
-
class ModelVersionArtifactFilter(
|
112
|
+
class ModelVersionArtifactFilter(BaseFilter):
|
159
113
|
"""Model version pipeline run links filter model."""
|
160
114
|
|
161
115
|
# Artifact name and type are not DB fields and need to be handled separately
|
162
116
|
FILTER_EXCLUDE_FIELDS = [
|
163
|
-
*
|
117
|
+
*BaseFilter.FILTER_EXCLUDE_FIELDS,
|
164
118
|
"artifact_name",
|
165
119
|
"only_data_artifacts",
|
166
120
|
"only_model_artifacts",
|
@@ -169,34 +123,16 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
|
|
169
123
|
"user",
|
170
124
|
]
|
171
125
|
CLI_EXCLUDE_FIELDS = [
|
172
|
-
*
|
126
|
+
*BaseFilter.CLI_EXCLUDE_FIELDS,
|
173
127
|
"only_data_artifacts",
|
174
128
|
"only_model_artifacts",
|
175
129
|
"only_deployment_artifacts",
|
176
130
|
"has_custom_name",
|
177
|
-
"model_id",
|
178
131
|
"model_version_id",
|
179
|
-
"user_id",
|
180
|
-
"workspace_id",
|
181
132
|
"updated",
|
182
133
|
"id",
|
183
134
|
]
|
184
135
|
|
185
|
-
workspace_id: Optional[Union[UUID, str]] = Field(
|
186
|
-
default=None,
|
187
|
-
description="The workspace of the Model Version",
|
188
|
-
union_mode="left_to_right",
|
189
|
-
)
|
190
|
-
user_id: Optional[Union[UUID, str]] = Field(
|
191
|
-
default=None,
|
192
|
-
description="The user of the Model Version",
|
193
|
-
union_mode="left_to_right",
|
194
|
-
)
|
195
|
-
model_id: Optional[Union[UUID, str]] = Field(
|
196
|
-
default=None,
|
197
|
-
description="Filter by model ID",
|
198
|
-
union_mode="left_to_right",
|
199
|
-
)
|
200
136
|
model_version_id: Optional[Union[UUID, str]] = Field(
|
201
137
|
default=None,
|
202
138
|
description="Filter by model version ID",
|
@@ -236,7 +172,7 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
|
|
236
172
|
"""
|
237
173
|
custom_filters = super().get_custom_filters()
|
238
174
|
|
239
|
-
from sqlmodel import and_
|
175
|
+
from sqlmodel import and_, col
|
240
176
|
|
241
177
|
from zenml.zen_stores.schemas import (
|
242
178
|
ArtifactSchema,
|
@@ -262,20 +198,27 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
|
|
262
198
|
|
263
199
|
if self.only_data_artifacts:
|
264
200
|
data_artifact_filter = and_(
|
265
|
-
ModelVersionArtifactSchema.
|
266
|
-
|
201
|
+
ModelVersionArtifactSchema.artifact_version_id
|
202
|
+
== ArtifactVersionSchema.id,
|
203
|
+
col(ArtifactVersionSchema.type).not_in(
|
204
|
+
["ServiceArtifact", "ModelArtifact"]
|
205
|
+
),
|
267
206
|
)
|
268
207
|
custom_filters.append(data_artifact_filter)
|
269
208
|
|
270
209
|
if self.only_model_artifacts:
|
271
210
|
model_artifact_filter = and_(
|
272
|
-
ModelVersionArtifactSchema.
|
211
|
+
ModelVersionArtifactSchema.artifact_version_id
|
212
|
+
== ArtifactVersionSchema.id,
|
213
|
+
ArtifactVersionSchema.type == "ModelArtifact",
|
273
214
|
)
|
274
215
|
custom_filters.append(model_artifact_filter)
|
275
216
|
|
276
217
|
if self.only_deployment_artifacts:
|
277
218
|
deployment_artifact_filter = and_(
|
278
|
-
ModelVersionArtifactSchema.
|
219
|
+
ModelVersionArtifactSchema.artifact_version_id
|
220
|
+
== ArtifactVersionSchema.id,
|
221
|
+
ArtifactVersionSchema.type == "ServiceArtifact",
|
279
222
|
)
|
280
223
|
custom_filters.append(deployment_artifact_filter)
|
281
224
|
|
@@ -23,23 +23,19 @@ from zenml.enums import GenericFilterOps
|
|
23
23
|
from zenml.models.v2.base.base import (
|
24
24
|
BaseDatedResponseBody,
|
25
25
|
BaseIdentifiedResponse,
|
26
|
+
BaseRequest,
|
26
27
|
BaseResponseMetadata,
|
27
28
|
BaseResponseResources,
|
28
29
|
)
|
29
|
-
from zenml.models.v2.base.filter import StrFilter
|
30
|
-
from zenml.models.v2.base.scoped import (
|
31
|
-
WorkspaceScopedFilter,
|
32
|
-
WorkspaceScopedRequest,
|
33
|
-
)
|
30
|
+
from zenml.models.v2.base.filter import BaseFilter, StrFilter
|
34
31
|
from zenml.models.v2.core.pipeline_run import PipelineRunResponse
|
35
32
|
|
36
33
|
# ------------------ Request Model ------------------
|
37
34
|
|
38
35
|
|
39
|
-
class ModelVersionPipelineRunRequest(
|
36
|
+
class ModelVersionPipelineRunRequest(BaseRequest):
|
40
37
|
"""Request model for links between model versions and pipeline runs."""
|
41
38
|
|
42
|
-
model: UUID
|
43
39
|
model_version: UUID
|
44
40
|
pipeline_run: UUID
|
45
41
|
|
@@ -62,7 +58,6 @@ class ModelVersionPipelineRunRequest(WorkspaceScopedRequest):
|
|
62
58
|
class ModelVersionPipelineRunResponseBody(BaseDatedResponseBody):
|
63
59
|
"""Response body for links between model versions and pipeline runs."""
|
64
60
|
|
65
|
-
model: UUID
|
66
61
|
model_version: UUID
|
67
62
|
pipeline_run: PipelineRunResponse
|
68
63
|
|
@@ -88,16 +83,6 @@ class ModelVersionPipelineRunResponse(
|
|
88
83
|
):
|
89
84
|
"""Response model for links between model versions and pipeline runs."""
|
90
85
|
|
91
|
-
# Body and metadata properties
|
92
|
-
@property
|
93
|
-
def model(self) -> UUID:
|
94
|
-
"""The `model` property.
|
95
|
-
|
96
|
-
Returns:
|
97
|
-
the value of the property.
|
98
|
-
"""
|
99
|
-
return self.get_body().model
|
100
|
-
|
101
86
|
@property
|
102
87
|
def model_version(self) -> UUID:
|
103
88
|
"""The `model_version` property.
|
@@ -120,39 +105,21 @@ class ModelVersionPipelineRunResponse(
|
|
120
105
|
# ------------------ Filter Model ------------------
|
121
106
|
|
122
107
|
|
123
|
-
class ModelVersionPipelineRunFilter(
|
108
|
+
class ModelVersionPipelineRunFilter(BaseFilter):
|
124
109
|
"""Model version pipeline run links filter model."""
|
125
110
|
|
126
111
|
FILTER_EXCLUDE_FIELDS = [
|
127
|
-
*
|
112
|
+
*BaseFilter.FILTER_EXCLUDE_FIELDS,
|
128
113
|
"pipeline_run_name",
|
129
114
|
"user",
|
130
115
|
]
|
131
116
|
CLI_EXCLUDE_FIELDS = [
|
132
|
-
*
|
133
|
-
"model_id",
|
117
|
+
*BaseFilter.CLI_EXCLUDE_FIELDS,
|
134
118
|
"model_version_id",
|
135
|
-
"user_id",
|
136
|
-
"workspace_id",
|
137
119
|
"updated",
|
138
120
|
"id",
|
139
121
|
]
|
140
122
|
|
141
|
-
workspace_id: Optional[Union[UUID, str]] = Field(
|
142
|
-
default=None,
|
143
|
-
description="The workspace of the Model Version",
|
144
|
-
union_mode="left_to_right",
|
145
|
-
)
|
146
|
-
user_id: Optional[Union[UUID, str]] = Field(
|
147
|
-
default=None,
|
148
|
-
description="The user of the Model Version",
|
149
|
-
union_mode="left_to_right",
|
150
|
-
)
|
151
|
-
model_id: Optional[Union[UUID, str]] = Field(
|
152
|
-
default=None,
|
153
|
-
description="Filter by model ID",
|
154
|
-
union_mode="left_to_right",
|
155
|
-
)
|
156
123
|
model_version_id: Optional[Union[UUID, str]] = Field(
|
157
124
|
default=None,
|
158
125
|
description="Filter by model version ID",
|
@@ -488,6 +488,10 @@ class ServiceConnectorResponse(
|
|
488
488
|
):
|
489
489
|
"""Response model for service connectors."""
|
490
490
|
|
491
|
+
# Disable the warning for updating responses, because we update the
|
492
|
+
# service connector type in place
|
493
|
+
_warn_on_response_updates: bool = False
|
494
|
+
|
491
495
|
name: str = Field(
|
492
496
|
title="The service connector name.",
|
493
497
|
max_length=STR_FIELD_MAX_LENGTH,
|
@@ -118,3 +118,26 @@ class ServerModel(BaseModel):
|
|
118
118
|
# Local ZenML servers are identifiable by the fact that their
|
119
119
|
# server ID is the same as the local client (user) ID.
|
120
120
|
return self.id == GlobalConfiguration().user_id
|
121
|
+
|
122
|
+
|
123
|
+
class ServerLoadInfo(BaseModel):
|
124
|
+
"""Domain model for ZenML server load information."""
|
125
|
+
|
126
|
+
threads: int = Field(
|
127
|
+
title="Number of threads that the server is currently using."
|
128
|
+
)
|
129
|
+
|
130
|
+
db_connections_total: int = Field(
|
131
|
+
title="Total number of database connections (active and idle) that the "
|
132
|
+
"server currently has established."
|
133
|
+
)
|
134
|
+
|
135
|
+
db_connections_active: int = Field(
|
136
|
+
title="Number of database connections that the server is currently "
|
137
|
+
"actively using to make queries or transactions."
|
138
|
+
)
|
139
|
+
|
140
|
+
db_connections_overflow: int = Field(
|
141
|
+
title="Number of overflow database connections that the server is "
|
142
|
+
"currently actively using to make queries or transactions."
|
143
|
+
)
|
@@ -14,12 +14,12 @@
|
|
14
14
|
"""Utilities for creating step runs."""
|
15
15
|
|
16
16
|
from datetime import datetime
|
17
|
-
from typing import TYPE_CHECKING, Dict, List,
|
17
|
+
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
|
18
18
|
|
19
19
|
from zenml.client import Client
|
20
|
-
from zenml.config.step_configurations import
|
20
|
+
from zenml.config.step_configurations import Step
|
21
21
|
from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
|
22
|
-
from zenml.enums import
|
22
|
+
from zenml.enums import ExecutionStatus
|
23
23
|
from zenml.logger import get_logger
|
24
24
|
from zenml.model.utils import link_artifact_version_to_model_version
|
25
25
|
from zenml.models import (
|
@@ -344,7 +344,6 @@ def create_cached_step_runs(
|
|
344
344
|
if model_version := step_model_version or pipeline_model_version:
|
345
345
|
link_output_artifacts_to_model_version(
|
346
346
|
artifacts=step_run.outputs,
|
347
|
-
output_configurations=step_run.config.outputs,
|
348
347
|
model_version=model_version,
|
349
348
|
)
|
350
349
|
|
@@ -542,10 +541,7 @@ def link_pipeline_run_to_model_version(
|
|
542
541
|
client = Client()
|
543
542
|
client.zen_store.create_model_version_pipeline_run_link(
|
544
543
|
ModelVersionPipelineRunRequest(
|
545
|
-
user=client.active_user.id,
|
546
|
-
workspace=client.active_workspace.id,
|
547
544
|
pipeline_run=pipeline_run.id,
|
548
|
-
model=model_version.model.id,
|
549
545
|
model_version=model_version.id,
|
550
546
|
)
|
551
547
|
)
|
@@ -553,26 +549,17 @@ def link_pipeline_run_to_model_version(
|
|
553
549
|
|
554
550
|
def link_output_artifacts_to_model_version(
|
555
551
|
artifacts: Dict[str, List[ArtifactVersionResponse]],
|
556
|
-
output_configurations: Mapping[str, ArtifactConfiguration],
|
557
552
|
model_version: ModelVersionResponse,
|
558
553
|
) -> None:
|
559
554
|
"""Link the outputs of a step run to a model version.
|
560
555
|
|
561
556
|
Args:
|
562
557
|
artifacts: The step output artifacts.
|
563
|
-
output_configurations: The output configurations for the step.
|
564
558
|
model_version: The model version to link.
|
565
559
|
"""
|
566
|
-
for
|
560
|
+
for output_artifacts in artifacts.values():
|
567
561
|
for output_artifact in output_artifacts:
|
568
|
-
artifact_config = None
|
569
|
-
if output_artifact.save_type == ArtifactSaveType.STEP_OUTPUT and (
|
570
|
-
output_config := output_configurations.get(output_name, None)
|
571
|
-
):
|
572
|
-
artifact_config = output_config.artifact_config
|
573
|
-
|
574
562
|
link_artifact_version_to_model_version(
|
575
563
|
artifact_version=output_artifact,
|
576
564
|
model_version=model_version,
|
577
|
-
artifact_config=artifact_config,
|
578
565
|
)
|
@@ -247,7 +247,6 @@ class StepRunner:
|
|
247
247
|
artifacts={
|
248
248
|
k: [v] for k, v in output_artifacts.items()
|
249
249
|
},
|
250
|
-
output_configurations=step_run.config.outputs,
|
251
250
|
model_version=model_version,
|
252
251
|
)
|
253
252
|
finally:
|
@@ -577,9 +576,11 @@ class StepRunner:
|
|
577
576
|
uri = output_artifact_uris[output_name]
|
578
577
|
artifact_config = output_annotations[output_name].artifact_config
|
579
578
|
|
579
|
+
artifact_type = None
|
580
580
|
if artifact_config is not None:
|
581
581
|
has_custom_name = bool(artifact_config.name)
|
582
582
|
version = artifact_config.version
|
583
|
+
artifact_type = artifact_config.artifact_type
|
583
584
|
else:
|
584
585
|
has_custom_name, version = False, None
|
585
586
|
|
@@ -605,6 +606,7 @@ class StepRunner:
|
|
605
606
|
data=return_value,
|
606
607
|
materializer_class=materializer_class,
|
607
608
|
uri=uri,
|
609
|
+
artifact_type=artifact_type,
|
608
610
|
store_metadata=artifact_metadata_enabled,
|
609
611
|
store_visualizations=artifact_visualization_enabled,
|
610
612
|
has_custom_name=has_custom_name,
|
@@ -3,6 +3,123 @@ Helpers for environment variables configured in ZenML deployments and secrets st
|
|
3
3
|
*/}}
|
4
4
|
|
5
5
|
|
6
|
+
{{/*
|
7
|
+
ZenML store configuration options (non-secret values).
|
8
|
+
|
9
|
+
This template constructs a dictionary that is similar to the python values that
|
10
|
+
can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration
|
11
|
+
class. Only non-secret values are included in this dictionary.
|
12
|
+
|
13
|
+
The dictionary is then converted into deployment environment variables by other
|
14
|
+
templates and inserted where it is needed.
|
15
|
+
|
16
|
+
The input is taken from a .ZenML dict that is passed to the template and
|
17
|
+
contains the values configured in the values.yaml file for the ZenML server.
|
18
|
+
|
19
|
+
Args:
|
20
|
+
.ZenML: A dictionary with the ZenML configuration values configured for the
|
21
|
+
ZenML server.
|
22
|
+
Returns:
|
23
|
+
A dictionary with the non-secret values configured for the ZenML store.
|
24
|
+
*/}}
|
25
|
+
{{- define "zenml.storeConfigurationAttrs" -}}
|
26
|
+
{{- if .ZenML.database.url }}
|
27
|
+
type: sql
|
28
|
+
ssl_verify_server_cert: {{ .ZenML.database.sslVerifyServerCert | default "false" | quote }}
|
29
|
+
{{- if .ZenML.database.backupStrategy }}
|
30
|
+
backup_strategy: {{ .ZenML.database.backupStrategy | quote }}
|
31
|
+
{{- if eq .ZenML.database.backupStrategy "database" }}
|
32
|
+
backup_database: {{ .ZenML.database.backupDatabase | quote }}
|
33
|
+
{{- else if eq .ZenML.database.backupStrategy "dump-file" }}
|
34
|
+
backup_directory: "/backups"
|
35
|
+
{{- end }}
|
36
|
+
{{- end }}
|
37
|
+
{{- if .ZenML.database.poolSize }}
|
38
|
+
pool_size: {{ .ZenML.database.poolSize | quote }}
|
39
|
+
{{- end }}
|
40
|
+
{{- if .ZenML.database.maxOverflow }}
|
41
|
+
max_overflow: {{ .ZenML.database.maxOverflow | quote }}
|
42
|
+
{{- end }}
|
43
|
+
{{- end }}
|
44
|
+
{{- end }}
|
45
|
+
|
46
|
+
|
47
|
+
{{/*
|
48
|
+
ZenML store configuration options (secret values).
|
49
|
+
|
50
|
+
This template constructs a dictionary that is similar to the python values that
|
51
|
+
can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration
|
52
|
+
class. Only secret values are included in this dictionary.
|
53
|
+
|
54
|
+
The dictionary is then converted into deployment environment variables by other
|
55
|
+
templates and inserted where it is needed.
|
56
|
+
|
57
|
+
The input is taken from a .ZenML dict that is passed to the template and
|
58
|
+
contains the values configured in the values.yaml file for the ZenML server.
|
59
|
+
|
60
|
+
Args:
|
61
|
+
.ZenML: A dictionary with the ZenML configuration values configured for the
|
62
|
+
ZenML server.
|
63
|
+
Returns:
|
64
|
+
A dictionary with the secret values configured for the ZenML store.
|
65
|
+
*/}}
|
66
|
+
{{- define "zenml.storeSecretConfigurationAttrs" -}}
|
67
|
+
{{- if .ZenML.database.url }}
|
68
|
+
url: {{ .ZenML.database.url | quote }}
|
69
|
+
{{- if .ZenML.database.sslCa }}
|
70
|
+
ssl_ca: {{ .Files.Get .ZenML.database.sslCa }}
|
71
|
+
{{- end }}
|
72
|
+
{{- if .ZenML.database.sslCert }}
|
73
|
+
ssl_cert: {{ .Files.Get .ZenML.database.sslCert }}
|
74
|
+
{{- end }}
|
75
|
+
{{- if .ZenML.database.sslKey }}
|
76
|
+
ssl_key: {{ .Files.Get .ZenML.database.sslKey }}
|
77
|
+
{{- end }}
|
78
|
+
{{- end }}
|
79
|
+
{{- end }}
|
80
|
+
|
81
|
+
|
82
|
+
{{/*
|
83
|
+
Store configuration environment variables (non-secret values).
|
84
|
+
|
85
|
+
Passes the .Values.zenml dict as input to the `zenml.storeConfigurationAttrs`
|
86
|
+
template and converts the output into a dictionary of environment variables that
|
87
|
+
need to be configured for the store.
|
88
|
+
|
89
|
+
Args:
|
90
|
+
.Values: The values.yaml file for the ZenML deployment.
|
91
|
+
Returns:
|
92
|
+
A dictionary with the non-secret environment variables that are configured for
|
93
|
+
the store (i.e. keys starting with `ZENML_STORE_`).
|
94
|
+
*/}}
|
95
|
+
{{- define "zenml.storeEnvVariables" -}}
|
96
|
+
{{ $zenml := dict "ZenML" .Values.zenml }}
|
97
|
+
{{- range $k, $v := include "zenml.storeConfigurationAttrs" $zenml | fromYaml }}
|
98
|
+
ZENML_STORE_{{ $k | upper }}: {{ $v | quote }}
|
99
|
+
{{- end }}
|
100
|
+
{{- end }}
|
101
|
+
|
102
|
+
|
103
|
+
{{/*
|
104
|
+
Store configuration environment variables (secret values).
|
105
|
+
|
106
|
+
Passes the .Values.zenml dict as input to the `zenml.storeSecretConfigurationAttrs`
|
107
|
+
template and converts the output into a dictionary of environment variables that
|
108
|
+
need to be configured for the store.
|
109
|
+
|
110
|
+
Args:
|
111
|
+
.Values: The values.yaml file for the ZenML deployment.
|
112
|
+
Returns:
|
113
|
+
A dictionary with the secret environment variables that are configured for
|
114
|
+
the store (i.e. keys starting with `ZENML_STORE_`).
|
115
|
+
*/}}
|
116
|
+
{{- define "zenml.storeSecretEnvVariables" -}}
|
117
|
+
{{ $zenml := dict "ZenML" .Values.zenml }}
|
118
|
+
{{- range $k, $v := include "zenml.storeSecretConfigurationAttrs" $zenml | fromYaml }}
|
119
|
+
ZENML_STORE_{{ $k | upper }}: {{ $v | quote }}
|
120
|
+
{{- end }}
|
121
|
+
{{- end }}
|
122
|
+
|
6
123
|
{{/*
|
7
124
|
ZenML server configuration options (non-secret values).
|
8
125
|
|
@@ -79,20 +79,9 @@ spec:
|
|
79
79
|
{{- end }}
|
80
80
|
- name: ZENML_DEFAULT_PROJECT_NAME
|
81
81
|
value: {{ .Values.zenml.defaultProject | quote }}
|
82
|
-
-
|
83
|
-
|
84
|
-
|
85
|
-
value: {{ .Values.zenml.database.sslVerifyServerCert | default "false" | quote }}
|
86
|
-
{{- if .Values.zenml.database.backupStrategy }}
|
87
|
-
- name: ZENML_STORE_BACKUP_STRATEGY
|
88
|
-
value: {{ .Values.zenml.database.backupStrategy | quote }}
|
89
|
-
{{- if eq .Values.zenml.database.backupStrategy "database" }}
|
90
|
-
- name: ZENML_STORE_BACKUP_DATABASE
|
91
|
-
value: {{ .Values.zenml.database.backupDatabase | quote }}
|
92
|
-
{{- else if eq .Values.zenml.database.backupStrategy "dump-file" }}
|
93
|
-
- name: ZENML_STORE_BACKUP_DIRECTORY
|
94
|
-
value: /backups
|
95
|
-
{{- end }}
|
82
|
+
{{- range $k, $v := include "zenml.storeEnvVariables" . | fromYaml }}
|
83
|
+
- name: {{ $k }}
|
84
|
+
value: {{ $v | quote }}
|
96
85
|
{{- end }}
|
97
86
|
{{- range $k, $v := include "zenml.serverEnvVariables" . | fromYaml }}
|
98
87
|
- name: {{ $k }}
|