zenml-nightly 0.70.0.dev20241120__py3-none-any.whl → 0.70.0.dev20241125__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. zenml/VERSION +1 -1
  2. zenml/artifacts/artifact_config.py +32 -4
  3. zenml/artifacts/utils.py +12 -24
  4. zenml/cli/base.py +1 -1
  5. zenml/client.py +4 -19
  6. zenml/constants.py +1 -0
  7. zenml/integrations/kubernetes/orchestrators/kube_utils.py +8 -7
  8. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +52 -1
  9. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -1
  10. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +3 -3
  11. zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +2 -1
  12. zenml/model/utils.py +0 -24
  13. zenml/models/__init__.py +6 -1
  14. zenml/models/v2/core/artifact_version.py +25 -2
  15. zenml/models/v2/core/model_version.py +0 -4
  16. zenml/models/v2/core/model_version_artifact.py +19 -76
  17. zenml/models/v2/core/model_version_pipeline_run.py +6 -39
  18. zenml/models/v2/core/service_connector.py +4 -0
  19. zenml/models/v2/misc/server_models.py +23 -0
  20. zenml/orchestrators/step_launcher.py +0 -1
  21. zenml/orchestrators/step_run_utils.py +4 -17
  22. zenml/orchestrators/step_runner.py +3 -1
  23. zenml/zen_server/deploy/helm/templates/_environment.tpl +117 -0
  24. zenml/zen_server/deploy/helm/templates/server-db-job.yaml +3 -14
  25. zenml/zen_server/deploy/helm/templates/server-deployment.yaml +16 -4
  26. zenml/zen_server/deploy/helm/templates/server-secret.yaml +2 -17
  27. zenml/zen_server/routers/model_versions_endpoints.py +59 -0
  28. zenml/zen_server/routers/server_endpoints.py +47 -0
  29. zenml/zen_server/routers/workspaces_endpoints.py +0 -130
  30. zenml/zen_server/zen_server_api.py +45 -6
  31. zenml/zen_stores/base_zen_store.py +2 -1
  32. zenml/zen_stores/migrations/utils.py +40 -24
  33. zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +118 -0
  34. zenml/zen_stores/rest_zen_store.py +42 -5
  35. zenml/zen_stores/schemas/model_schemas.py +10 -94
  36. zenml/zen_stores/schemas/user_schemas.py +0 -8
  37. zenml/zen_stores/schemas/workspace_schemas.py +0 -14
  38. {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/METADATA +1 -1
  39. {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/RECORD +42 -41
  40. {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/LICENSE +0 -0
  41. {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/WHEEL +0 -0
  42. {zenml_nightly-0.70.0.dev20241120.dist-info → zenml_nightly-0.70.0.dev20241125.dist-info}/entry_points.txt +0 -0
@@ -37,7 +37,7 @@ from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
37
37
  from zenml.enums import ArtifactSaveType, ArtifactType, GenericFilterOps
38
38
  from zenml.logger import get_logger
39
39
  from zenml.metadata.metadata_types import MetadataType
40
- from zenml.models.v2.base.filter import StrFilter
40
+ from zenml.models.v2.base.filter import FilterGenerator, StrFilter
41
41
  from zenml.models.v2.base.scoped import (
42
42
  WorkspaceScopedRequest,
43
43
  WorkspaceScopedResponse,
@@ -474,6 +474,7 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
474
474
  "user",
475
475
  "model",
476
476
  "pipeline_run",
477
+ "model_version_id",
477
478
  "run_metadata",
478
479
  ]
479
480
  artifact_id: Optional[Union[UUID, str]] = Field(
@@ -525,6 +526,11 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
525
526
  description="User that produced this artifact",
526
527
  union_mode="left_to_right",
527
528
  )
529
+ model_version_id: Optional[Union[UUID, str]] = Field(
530
+ default=None,
531
+ description="ID of the model version that is associated with this artifact version.",
532
+ union_mode="left_to_right",
533
+ )
528
534
  only_unused: Optional[bool] = Field(
529
535
  default=False, description="Filter only for unused artifacts"
530
536
  )
@@ -568,6 +574,7 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
568
574
  ArtifactVersionSchema,
569
575
  ModelSchema,
570
576
  ModelVersionArtifactSchema,
577
+ ModelVersionSchema,
571
578
  PipelineRunSchema,
572
579
  RunMetadataSchema,
573
580
  StepRunInputArtifactSchema,
@@ -600,6 +607,20 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
600
607
  )
601
608
  custom_filters.append(unused_filter)
602
609
 
610
+ if self.model_version_id:
611
+ value, operator = self._resolve_operator(self.model_version_id)
612
+
613
+ model_version_filter = and_(
614
+ ArtifactVersionSchema.id
615
+ == ModelVersionArtifactSchema.artifact_version_id,
616
+ ModelVersionArtifactSchema.model_version_id
617
+ == ModelVersionSchema.id,
618
+ FilterGenerator(ModelVersionSchema)
619
+ .define_filter(column="id", value=value, operator=operator)
620
+ .generate_query_conditions(ModelVersionSchema),
621
+ )
622
+ custom_filters.append(model_version_filter)
623
+
603
624
  if self.has_custom_name is not None:
604
625
  custom_name_filter = and_(
605
626
  ArtifactVersionSchema.artifact_id == ArtifactSchema.id,
@@ -622,7 +643,9 @@ class ArtifactVersionFilter(WorkspaceScopedTaggableFilter):
622
643
  model_filter = and_(
623
644
  ArtifactVersionSchema.id
624
645
  == ModelVersionArtifactSchema.artifact_version_id,
625
- ModelVersionArtifactSchema.model_id == ModelSchema.id,
646
+ ModelVersionArtifactSchema.model_version_id
647
+ == ModelVersionSchema.id,
648
+ ModelVersionSchema.model_id == ModelSchema.id,
626
649
  self.generate_name_or_id_query_conditions(
627
650
  value=self.model, table=ModelSchema
628
651
  ),
@@ -576,10 +576,6 @@ class ModelVersionResponse(
576
576
  force=force,
577
577
  )
578
578
 
579
- # TODO in https://zenml.atlassian.net/browse/OSS-2433
580
- # def generate_model_card(self, template_name: str) -> str:
581
- # """Return HTML/PDF based on input template"""
582
-
583
579
 
584
580
  # ------------------ Filter Model ------------------
585
581
 
@@ -16,20 +16,17 @@
16
16
  from typing import TYPE_CHECKING, List, Optional, Union
17
17
  from uuid import UUID
18
18
 
19
- from pydantic import ConfigDict, Field, model_validator
19
+ from pydantic import ConfigDict, Field
20
20
 
21
21
  from zenml.enums import GenericFilterOps
22
22
  from zenml.models.v2.base.base import (
23
23
  BaseDatedResponseBody,
24
24
  BaseIdentifiedResponse,
25
+ BaseRequest,
25
26
  BaseResponseMetadata,
26
27
  BaseResponseResources,
27
28
  )
28
- from zenml.models.v2.base.filter import StrFilter
29
- from zenml.models.v2.base.scoped import (
30
- WorkspaceScopedFilter,
31
- WorkspaceScopedRequest,
32
- )
29
+ from zenml.models.v2.base.filter import BaseFilter, StrFilter
33
30
 
34
31
  if TYPE_CHECKING:
35
32
  from sqlalchemy.sql.elements import ColumnElement
@@ -40,14 +37,11 @@ if TYPE_CHECKING:
40
37
  # ------------------ Request Model ------------------
41
38
 
42
39
 
43
- class ModelVersionArtifactRequest(WorkspaceScopedRequest):
40
+ class ModelVersionArtifactRequest(BaseRequest):
44
41
  """Request model for links between model versions and artifacts."""
45
42
 
46
- model: UUID
47
43
  model_version: UUID
48
44
  artifact_version: UUID
49
- is_model_artifact: bool = False
50
- is_deployment_artifact: bool = False
51
45
 
52
46
  # TODO: In Pydantic v2, the `model_` is a protected namespaces for all
53
47
  # fields defined under base models. If not handled, this raises a warning.
@@ -57,15 +51,6 @@ class ModelVersionArtifactRequest(WorkspaceScopedRequest):
57
51
  # careful we might overwrite some fields protected by pydantic.
58
52
  model_config = ConfigDict(protected_namespaces=())
59
53
 
60
- @model_validator(mode="after")
61
- def _validate_is_endpoint_artifact(self) -> "ModelVersionArtifactRequest":
62
- if self.is_model_artifact and self.is_deployment_artifact:
63
- raise ValueError(
64
- "Artifact cannot be a model artifact and deployment artifact "
65
- "at the same time."
66
- )
67
- return self
68
-
69
54
 
70
55
  # ------------------ Update Model ------------------
71
56
 
@@ -77,11 +62,8 @@ class ModelVersionArtifactRequest(WorkspaceScopedRequest):
77
62
  class ModelVersionArtifactResponseBody(BaseDatedResponseBody):
78
63
  """Response body for links between model versions and artifacts."""
79
64
 
80
- model: UUID
81
65
  model_version: UUID
82
66
  artifact_version: "ArtifactVersionResponse"
83
- is_model_artifact: bool = False
84
- is_deployment_artifact: bool = False
85
67
 
86
68
  # TODO: In Pydantic v2, the `model_` is a protected namespaces for all
87
69
  # fields defined under base models. If not handled, this raises a warning.
@@ -105,16 +87,6 @@ class ModelVersionArtifactResponse(
105
87
  ):
106
88
  """Response model for links between model versions and artifacts."""
107
89
 
108
- # Body and metadata properties
109
- @property
110
- def model(self) -> UUID:
111
- """The `model` property.
112
-
113
- Returns:
114
- the value of the property.
115
- """
116
- return self.get_body().model
117
-
118
90
  @property
119
91
  def model_version(self) -> UUID:
120
92
  """The `model_version` property.
@@ -133,34 +105,16 @@ class ModelVersionArtifactResponse(
133
105
  """
134
106
  return self.get_body().artifact_version
135
107
 
136
- @property
137
- def is_model_artifact(self) -> bool:
138
- """The `is_model_artifact` property.
139
-
140
- Returns:
141
- the value of the property.
142
- """
143
- return self.get_body().is_model_artifact
144
-
145
- @property
146
- def is_deployment_artifact(self) -> bool:
147
- """The `is_deployment_artifact` property.
148
-
149
- Returns:
150
- the value of the property.
151
- """
152
- return self.get_body().is_deployment_artifact
153
-
154
108
 
155
109
  # ------------------ Filter Model ------------------
156
110
 
157
111
 
158
- class ModelVersionArtifactFilter(WorkspaceScopedFilter):
112
+ class ModelVersionArtifactFilter(BaseFilter):
159
113
  """Model version pipeline run links filter model."""
160
114
 
161
115
  # Artifact name and type are not DB fields and need to be handled separately
162
116
  FILTER_EXCLUDE_FIELDS = [
163
- *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
117
+ *BaseFilter.FILTER_EXCLUDE_FIELDS,
164
118
  "artifact_name",
165
119
  "only_data_artifacts",
166
120
  "only_model_artifacts",
@@ -169,34 +123,16 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
169
123
  "user",
170
124
  ]
171
125
  CLI_EXCLUDE_FIELDS = [
172
- *WorkspaceScopedFilter.CLI_EXCLUDE_FIELDS,
126
+ *BaseFilter.CLI_EXCLUDE_FIELDS,
173
127
  "only_data_artifacts",
174
128
  "only_model_artifacts",
175
129
  "only_deployment_artifacts",
176
130
  "has_custom_name",
177
- "model_id",
178
131
  "model_version_id",
179
- "user_id",
180
- "workspace_id",
181
132
  "updated",
182
133
  "id",
183
134
  ]
184
135
 
185
- workspace_id: Optional[Union[UUID, str]] = Field(
186
- default=None,
187
- description="The workspace of the Model Version",
188
- union_mode="left_to_right",
189
- )
190
- user_id: Optional[Union[UUID, str]] = Field(
191
- default=None,
192
- description="The user of the Model Version",
193
- union_mode="left_to_right",
194
- )
195
- model_id: Optional[Union[UUID, str]] = Field(
196
- default=None,
197
- description="Filter by model ID",
198
- union_mode="left_to_right",
199
- )
200
136
  model_version_id: Optional[Union[UUID, str]] = Field(
201
137
  default=None,
202
138
  description="Filter by model version ID",
@@ -236,7 +172,7 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
236
172
  """
237
173
  custom_filters = super().get_custom_filters()
238
174
 
239
- from sqlmodel import and_
175
+ from sqlmodel import and_, col
240
176
 
241
177
  from zenml.zen_stores.schemas import (
242
178
  ArtifactSchema,
@@ -262,20 +198,27 @@ class ModelVersionArtifactFilter(WorkspaceScopedFilter):
262
198
 
263
199
  if self.only_data_artifacts:
264
200
  data_artifact_filter = and_(
265
- ModelVersionArtifactSchema.is_model_artifact.is_(False), # type: ignore[attr-defined]
266
- ModelVersionArtifactSchema.is_deployment_artifact.is_(False), # type: ignore[attr-defined]
201
+ ModelVersionArtifactSchema.artifact_version_id
202
+ == ArtifactVersionSchema.id,
203
+ col(ArtifactVersionSchema.type).not_in(
204
+ ["ServiceArtifact", "ModelArtifact"]
205
+ ),
267
206
  )
268
207
  custom_filters.append(data_artifact_filter)
269
208
 
270
209
  if self.only_model_artifacts:
271
210
  model_artifact_filter = and_(
272
- ModelVersionArtifactSchema.is_model_artifact.is_(True), # type: ignore[attr-defined]
211
+ ModelVersionArtifactSchema.artifact_version_id
212
+ == ArtifactVersionSchema.id,
213
+ ArtifactVersionSchema.type == "ModelArtifact",
273
214
  )
274
215
  custom_filters.append(model_artifact_filter)
275
216
 
276
217
  if self.only_deployment_artifacts:
277
218
  deployment_artifact_filter = and_(
278
- ModelVersionArtifactSchema.is_deployment_artifact.is_(True), # type: ignore[attr-defined]
219
+ ModelVersionArtifactSchema.artifact_version_id
220
+ == ArtifactVersionSchema.id,
221
+ ArtifactVersionSchema.type == "ServiceArtifact",
279
222
  )
280
223
  custom_filters.append(deployment_artifact_filter)
281
224
 
@@ -23,23 +23,19 @@ from zenml.enums import GenericFilterOps
23
23
  from zenml.models.v2.base.base import (
24
24
  BaseDatedResponseBody,
25
25
  BaseIdentifiedResponse,
26
+ BaseRequest,
26
27
  BaseResponseMetadata,
27
28
  BaseResponseResources,
28
29
  )
29
- from zenml.models.v2.base.filter import StrFilter
30
- from zenml.models.v2.base.scoped import (
31
- WorkspaceScopedFilter,
32
- WorkspaceScopedRequest,
33
- )
30
+ from zenml.models.v2.base.filter import BaseFilter, StrFilter
34
31
  from zenml.models.v2.core.pipeline_run import PipelineRunResponse
35
32
 
36
33
  # ------------------ Request Model ------------------
37
34
 
38
35
 
39
- class ModelVersionPipelineRunRequest(WorkspaceScopedRequest):
36
+ class ModelVersionPipelineRunRequest(BaseRequest):
40
37
  """Request model for links between model versions and pipeline runs."""
41
38
 
42
- model: UUID
43
39
  model_version: UUID
44
40
  pipeline_run: UUID
45
41
 
@@ -62,7 +58,6 @@ class ModelVersionPipelineRunRequest(WorkspaceScopedRequest):
62
58
  class ModelVersionPipelineRunResponseBody(BaseDatedResponseBody):
63
59
  """Response body for links between model versions and pipeline runs."""
64
60
 
65
- model: UUID
66
61
  model_version: UUID
67
62
  pipeline_run: PipelineRunResponse
68
63
 
@@ -88,16 +83,6 @@ class ModelVersionPipelineRunResponse(
88
83
  ):
89
84
  """Response model for links between model versions and pipeline runs."""
90
85
 
91
- # Body and metadata properties
92
- @property
93
- def model(self) -> UUID:
94
- """The `model` property.
95
-
96
- Returns:
97
- the value of the property.
98
- """
99
- return self.get_body().model
100
-
101
86
  @property
102
87
  def model_version(self) -> UUID:
103
88
  """The `model_version` property.
@@ -120,39 +105,21 @@ class ModelVersionPipelineRunResponse(
120
105
  # ------------------ Filter Model ------------------
121
106
 
122
107
 
123
- class ModelVersionPipelineRunFilter(WorkspaceScopedFilter):
108
+ class ModelVersionPipelineRunFilter(BaseFilter):
124
109
  """Model version pipeline run links filter model."""
125
110
 
126
111
  FILTER_EXCLUDE_FIELDS = [
127
- *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
112
+ *BaseFilter.FILTER_EXCLUDE_FIELDS,
128
113
  "pipeline_run_name",
129
114
  "user",
130
115
  ]
131
116
  CLI_EXCLUDE_FIELDS = [
132
- *WorkspaceScopedFilter.CLI_EXCLUDE_FIELDS,
133
- "model_id",
117
+ *BaseFilter.CLI_EXCLUDE_FIELDS,
134
118
  "model_version_id",
135
- "user_id",
136
- "workspace_id",
137
119
  "updated",
138
120
  "id",
139
121
  ]
140
122
 
141
- workspace_id: Optional[Union[UUID, str]] = Field(
142
- default=None,
143
- description="The workspace of the Model Version",
144
- union_mode="left_to_right",
145
- )
146
- user_id: Optional[Union[UUID, str]] = Field(
147
- default=None,
148
- description="The user of the Model Version",
149
- union_mode="left_to_right",
150
- )
151
- model_id: Optional[Union[UUID, str]] = Field(
152
- default=None,
153
- description="Filter by model ID",
154
- union_mode="left_to_right",
155
- )
156
123
  model_version_id: Optional[Union[UUID, str]] = Field(
157
124
  default=None,
158
125
  description="Filter by model version ID",
@@ -488,6 +488,10 @@ class ServiceConnectorResponse(
488
488
  ):
489
489
  """Response model for service connectors."""
490
490
 
491
+ # Disable the warning for updating responses, because we update the
492
+ # service connector type in place
493
+ _warn_on_response_updates: bool = False
494
+
491
495
  name: str = Field(
492
496
  title="The service connector name.",
493
497
  max_length=STR_FIELD_MAX_LENGTH,
@@ -118,3 +118,26 @@ class ServerModel(BaseModel):
118
118
  # Local ZenML servers are identifiable by the fact that their
119
119
  # server ID is the same as the local client (user) ID.
120
120
  return self.id == GlobalConfiguration().user_id
121
+
122
+
123
+ class ServerLoadInfo(BaseModel):
124
+ """Domain model for ZenML server load information."""
125
+
126
+ threads: int = Field(
127
+ title="Number of threads that the server is currently using."
128
+ )
129
+
130
+ db_connections_total: int = Field(
131
+ title="Total number of database connections (active and idle) that the "
132
+ "server currently has established."
133
+ )
134
+
135
+ db_connections_active: int = Field(
136
+ title="Number of database connections that the server is currently "
137
+ "actively using to make queries or transactions."
138
+ )
139
+
140
+ db_connections_overflow: int = Field(
141
+ title="Number of overflow database connections that the server is "
142
+ "currently actively using to make queries or transactions."
143
+ )
@@ -294,7 +294,6 @@ class StepLauncher:
294
294
  ):
295
295
  step_run_utils.link_output_artifacts_to_model_version(
296
296
  artifacts=step_run.outputs,
297
- output_configurations=step_run.config.outputs,
298
297
  model_version=model_version,
299
298
  )
300
299
 
@@ -14,12 +14,12 @@
14
14
  """Utilities for creating step runs."""
15
15
 
16
16
  from datetime import datetime
17
- from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple
17
+ from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple
18
18
 
19
19
  from zenml.client import Client
20
- from zenml.config.step_configurations import ArtifactConfiguration, Step
20
+ from zenml.config.step_configurations import Step
21
21
  from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
22
- from zenml.enums import ArtifactSaveType, ExecutionStatus
22
+ from zenml.enums import ExecutionStatus
23
23
  from zenml.logger import get_logger
24
24
  from zenml.model.utils import link_artifact_version_to_model_version
25
25
  from zenml.models import (
@@ -344,7 +344,6 @@ def create_cached_step_runs(
344
344
  if model_version := step_model_version or pipeline_model_version:
345
345
  link_output_artifacts_to_model_version(
346
346
  artifacts=step_run.outputs,
347
- output_configurations=step_run.config.outputs,
348
347
  model_version=model_version,
349
348
  )
350
349
 
@@ -542,10 +541,7 @@ def link_pipeline_run_to_model_version(
542
541
  client = Client()
543
542
  client.zen_store.create_model_version_pipeline_run_link(
544
543
  ModelVersionPipelineRunRequest(
545
- user=client.active_user.id,
546
- workspace=client.active_workspace.id,
547
544
  pipeline_run=pipeline_run.id,
548
- model=model_version.model.id,
549
545
  model_version=model_version.id,
550
546
  )
551
547
  )
@@ -553,26 +549,17 @@ def link_pipeline_run_to_model_version(
553
549
 
554
550
  def link_output_artifacts_to_model_version(
555
551
  artifacts: Dict[str, List[ArtifactVersionResponse]],
556
- output_configurations: Mapping[str, ArtifactConfiguration],
557
552
  model_version: ModelVersionResponse,
558
553
  ) -> None:
559
554
  """Link the outputs of a step run to a model version.
560
555
 
561
556
  Args:
562
557
  artifacts: The step output artifacts.
563
- output_configurations: The output configurations for the step.
564
558
  model_version: The model version to link.
565
559
  """
566
- for output_name, output_artifacts in artifacts.items():
560
+ for output_artifacts in artifacts.values():
567
561
  for output_artifact in output_artifacts:
568
- artifact_config = None
569
- if output_artifact.save_type == ArtifactSaveType.STEP_OUTPUT and (
570
- output_config := output_configurations.get(output_name, None)
571
- ):
572
- artifact_config = output_config.artifact_config
573
-
574
562
  link_artifact_version_to_model_version(
575
563
  artifact_version=output_artifact,
576
564
  model_version=model_version,
577
- artifact_config=artifact_config,
578
565
  )
@@ -247,7 +247,6 @@ class StepRunner:
247
247
  artifacts={
248
248
  k: [v] for k, v in output_artifacts.items()
249
249
  },
250
- output_configurations=step_run.config.outputs,
251
250
  model_version=model_version,
252
251
  )
253
252
  finally:
@@ -577,9 +576,11 @@ class StepRunner:
577
576
  uri = output_artifact_uris[output_name]
578
577
  artifact_config = output_annotations[output_name].artifact_config
579
578
 
579
+ artifact_type = None
580
580
  if artifact_config is not None:
581
581
  has_custom_name = bool(artifact_config.name)
582
582
  version = artifact_config.version
583
+ artifact_type = artifact_config.artifact_type
583
584
  else:
584
585
  has_custom_name, version = False, None
585
586
 
@@ -605,6 +606,7 @@ class StepRunner:
605
606
  data=return_value,
606
607
  materializer_class=materializer_class,
607
608
  uri=uri,
609
+ artifact_type=artifact_type,
608
610
  store_metadata=artifact_metadata_enabled,
609
611
  store_visualizations=artifact_visualization_enabled,
610
612
  has_custom_name=has_custom_name,
@@ -3,6 +3,123 @@ Helpers for environment variables configured in ZenML deployments and secrets st
3
3
  */}}
4
4
 
5
5
 
6
+ {{/*
7
+ ZenML store configuration options (non-secret values).
8
+
9
+ This template constructs a dictionary that is similar to the python values that
10
+ can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration
11
+ class. Only non-secret values are included in this dictionary.
12
+
13
+ The dictionary is then converted into deployment environment variables by other
14
+ templates and inserted where it is needed.
15
+
16
+ The input is taken from a .ZenML dict that is passed to the template and
17
+ contains the values configured in the values.yaml file for the ZenML server.
18
+
19
+ Args:
20
+ .ZenML: A dictionary with the ZenML configuration values configured for the
21
+ ZenML server.
22
+ Returns:
23
+ A dictionary with the non-secret values configured for the ZenML store.
24
+ */}}
25
+ {{- define "zenml.storeConfigurationAttrs" -}}
26
+ {{- if .ZenML.database.url }}
27
+ type: sql
28
+ ssl_verify_server_cert: {{ .ZenML.database.sslVerifyServerCert | default "false" | quote }}
29
+ {{- if .ZenML.database.backupStrategy }}
30
+ backup_strategy: {{ .ZenML.database.backupStrategy | quote }}
31
+ {{- if eq .ZenML.database.backupStrategy "database" }}
32
+ backup_database: {{ .ZenML.database.backupDatabase | quote }}
33
+ {{- else if eq .ZenML.database.backupStrategy "dump-file" }}
34
+ backup_directory: "/backups"
35
+ {{- end }}
36
+ {{- end }}
37
+ {{- if .ZenML.database.poolSize }}
38
+ pool_size: {{ .ZenML.database.poolSize | quote }}
39
+ {{- end }}
40
+ {{- if .ZenML.database.maxOverflow }}
41
+ max_overflow: {{ .ZenML.database.maxOverflow | quote }}
42
+ {{- end }}
43
+ {{- end }}
44
+ {{- end }}
45
+
46
+
47
+ {{/*
48
+ ZenML store configuration options (secret values).
49
+
50
+ This template constructs a dictionary that is similar to the python values that
51
+ can be configured in the zenml.zen_store.sql_zen_store.SqlZenStoreConfiguration
52
+ class. Only secret values are included in this dictionary.
53
+
54
+ The dictionary is then converted into deployment environment variables by other
55
+ templates and inserted where it is needed.
56
+
57
+ The input is taken from a .ZenML dict that is passed to the template and
58
+ contains the values configured in the values.yaml file for the ZenML server.
59
+
60
+ Args:
61
+ .ZenML: A dictionary with the ZenML configuration values configured for the
62
+ ZenML server.
63
+ Returns:
64
+ A dictionary with the secret values configured for the ZenML store.
65
+ */}}
66
+ {{- define "zenml.storeSecretConfigurationAttrs" -}}
67
+ {{- if .ZenML.database.url }}
68
+ url: {{ .ZenML.database.url | quote }}
69
+ {{- if .ZenML.database.sslCa }}
70
+ ssl_ca: {{ .Files.Get .ZenML.database.sslCa }}
71
+ {{- end }}
72
+ {{- if .ZenML.database.sslCert }}
73
+ ssl_cert: {{ .Files.Get .ZenML.database.sslCert }}
74
+ {{- end }}
75
+ {{- if .ZenML.database.sslKey }}
76
+ ssl_key: {{ .Files.Get .ZenML.database.sslKey }}
77
+ {{- end }}
78
+ {{- end }}
79
+ {{- end }}
80
+
81
+
82
+ {{/*
83
+ Store configuration environment variables (non-secret values).
84
+
85
+ Passes the .Values.zenml dict as input to the `zenml.storeConfigurationAttrs`
86
+ template and converts the output into a dictionary of environment variables that
87
+ need to be configured for the store.
88
+
89
+ Args:
90
+ .Values: The values.yaml file for the ZenML deployment.
91
+ Returns:
92
+ A dictionary with the non-secret environment variables that are configured for
93
+ the store (i.e. keys starting with `ZENML_STORE_`).
94
+ */}}
95
+ {{- define "zenml.storeEnvVariables" -}}
96
+ {{ $zenml := dict "ZenML" .Values.zenml }}
97
+ {{- range $k, $v := include "zenml.storeConfigurationAttrs" $zenml | fromYaml }}
98
+ ZENML_STORE_{{ $k | upper }}: {{ $v | quote }}
99
+ {{- end }}
100
+ {{- end }}
101
+
102
+
103
+ {{/*
104
+ Store configuration environment variables (secret values).
105
+
106
+ Passes the .Values.zenml dict as input to the `zenml.storeSecretConfigurationAttrs`
107
+ template and converts the output into a dictionary of environment variables that
108
+ need to be configured for the store.
109
+
110
+ Args:
111
+ .Values: The values.yaml file for the ZenML deployment.
112
+ Returns:
113
+ A dictionary with the secret environment variables that are configured for
114
+ the store (i.e. keys starting with `ZENML_STORE_`).
115
+ */}}
116
+ {{- define "zenml.storeSecretEnvVariables" -}}
117
+ {{ $zenml := dict "ZenML" .Values.zenml }}
118
+ {{- range $k, $v := include "zenml.storeSecretConfigurationAttrs" $zenml | fromYaml }}
119
+ ZENML_STORE_{{ $k | upper }}: {{ $v | quote }}
120
+ {{- end }}
121
+ {{- end }}
122
+
6
123
  {{/*
7
124
  ZenML server configuration options (non-secret values).
8
125
 
@@ -79,20 +79,9 @@ spec:
79
79
  {{- end }}
80
80
  - name: ZENML_DEFAULT_PROJECT_NAME
81
81
  value: {{ .Values.zenml.defaultProject | quote }}
82
- - name: ZENML_STORE_TYPE
83
- value: sql
84
- - name: ZENML_STORE_SSL_VERIFY_SERVER_CERT
85
- value: {{ .Values.zenml.database.sslVerifyServerCert | default "false" | quote }}
86
- {{- if .Values.zenml.database.backupStrategy }}
87
- - name: ZENML_STORE_BACKUP_STRATEGY
88
- value: {{ .Values.zenml.database.backupStrategy | quote }}
89
- {{- if eq .Values.zenml.database.backupStrategy "database" }}
90
- - name: ZENML_STORE_BACKUP_DATABASE
91
- value: {{ .Values.zenml.database.backupDatabase | quote }}
92
- {{- else if eq .Values.zenml.database.backupStrategy "dump-file" }}
93
- - name: ZENML_STORE_BACKUP_DIRECTORY
94
- value: /backups
95
- {{- end }}
82
+ {{- range $k, $v := include "zenml.storeEnvVariables" . | fromYaml }}
83
+ - name: {{ $k }}
84
+ value: {{ $v | quote }}
96
85
  {{- end }}
97
86
  {{- range $k, $v := include "zenml.serverEnvVariables" . | fromYaml }}
98
87
  - name: {{ $k }}