zenml-nightly 0.68.1.dev20241110__py3-none-any.whl → 0.68.1.dev20241112__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
zenml/VERSION CHANGED
@@ -1 +1 @@
1
- 0.68.1.dev20241110
1
+ 0.68.1.dev20241112
zenml/cli/base.py CHANGED
@@ -91,7 +91,7 @@ ZENML_PROJECT_TEMPLATES = dict(
91
91
  ),
92
92
  llm_finetuning=ZenMLProjectTemplateLocation(
93
93
  github_url="zenml-io/template-llm-finetuning",
94
- github_tag="2024.10.30", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
94
+ github_tag="2024.11.08", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
95
95
  ),
96
96
  )
97
97
 
zenml/client.py CHANGED
@@ -3426,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
3426
3426
  Args:
3427
3427
  id_or_prefix: The id or id prefix of the deployment.
3428
3428
  """
3429
- deployment = self.get_deployment(id_or_prefix=id_or_prefix)
3429
+ deployment = self.get_deployment(
3430
+ id_or_prefix=id_or_prefix, hydrate=False
3431
+ )
3430
3432
  self.zen_store.delete_deployment(deployment_id=deployment.id)
3431
3433
 
3432
3434
  # ------------------------------ Run templates -----------------------------
zenml/exceptions.py CHANGED
@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
168
168
  """Raised when trying to register an entity that already exists."""
169
169
 
170
170
 
171
+ class EntityCreationError(ZenMLBaseException, RuntimeError):
172
+ """Raised when failing to create an entity."""
173
+
174
+
171
175
  class ActionExistsError(EntityExistsError):
172
176
  """Raised when registering an action with a name that already exists."""
173
177
 
@@ -518,19 +518,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
518
518
  "pipeline_execution_arn": pipeline_execution_arn,
519
519
  }
520
520
 
521
- aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
522
-
523
- region_name, _, _ = dissect_pipeline_execution_arn(
524
- pipeline_execution_arn=pipeline_execution_arn
525
- )
526
-
527
- orchestrator_logs_url = (
528
- f"https://{region_name}.console.aws.amazon.com/"
529
- f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
530
- f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
531
- f"$3Dpipelines-{aws_run_id}-"
532
- )
533
- run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
534
521
  return run_metadata
535
522
 
536
523
  def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
@@ -673,7 +660,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
673
660
  return (
674
661
  f"https://{region_name}.console.aws.amazon.com/"
675
662
  f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
676
- f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
663
+ f"/$252Faws$252Fsagemaker$252FTrainingJobs$3FlogStreamNameFilter"
677
664
  f"$3Dpipelines-{execution_id}-"
678
665
  )
679
666
  except Exception as e:
@@ -390,10 +390,13 @@ class LightningOrchestrator(WheeledOrchestrator):
390
390
  studio.run(
391
391
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
392
392
  )
393
- studio.upload_file(
394
- env_file_path,
395
- remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
393
+ studio.upload_file(env_file_path)
394
+ time.sleep(6)
395
+ studio.run(
396
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
396
397
  )
398
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
399
+
397
400
  studio.run("pip install uv")
398
401
  logger.info(
399
402
  f"Installing requirements: {pipeline_requirements_to_string}"
@@ -481,16 +484,25 @@ class LightningOrchestrator(WheeledOrchestrator):
481
484
  code_path,
482
485
  remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
483
486
  )
487
+ time.sleep(6)
484
488
  studio.run(
485
489
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
486
490
  )
487
491
  logger.info(
488
492
  "Uploading wheel package and installing dependencies on main studio"
489
493
  )
490
- studio.upload_file(
491
- env_file_path,
492
- remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
494
+ # for some reason uploading file directly to /teamspace/studios/this_studio/.lightning_studio/.studiorc
495
+ # doesn't work, while other file names can be uploaded just fine.
496
+ # below is a workaround to copy the file to the correct location.
497
+ # we first upload it under a different name and then copy it to the
498
+ # right location.
499
+ studio.upload_file(env_file_path)
500
+ time.sleep(6)
501
+ studio.run(
502
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
493
503
  )
504
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
505
+
494
506
  studio.run("pip install uv")
495
507
  studio.run(f"uv pip install {requirements}")
496
508
  studio.run("pip install zenml")
@@ -557,13 +569,21 @@ class LightningOrchestrator(WheeledOrchestrator):
557
569
  studio.run(
558
570
  f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
559
571
  )
560
- studio.upload_file(code_path, remote_path=f"/zenml_codes/{filename}")
572
+ studio.upload_file(
573
+ code_path,
574
+ remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
575
+ )
576
+ time.sleep(6)
561
577
  studio.run(
562
578
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
563
579
  )
564
- studio.upload_file(
565
- env_file_path, remote_path=".lightning_studio/.studiorc"
580
+ studio.upload_file(env_file_path)
581
+ time.sleep(6)
582
+ studio.run(
583
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
566
584
  )
585
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
586
+
567
587
  studio.run("pip install uv")
568
588
  studio.run(f"uv pip install {details['requirements']}")
569
589
  studio.run("pip install zenml")
zenml/model/model.py CHANGED
@@ -14,7 +14,6 @@
14
14
  """Model user facing interface to pass into pipeline or step."""
15
15
 
16
16
  import datetime
17
- import time
18
17
  from typing import (
19
18
  TYPE_CHECKING,
20
19
  Any,
@@ -28,7 +27,6 @@ from uuid import UUID
28
27
 
29
28
  from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
30
29
 
31
- from zenml.constants import MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION
32
30
  from zenml.enums import MetadataResourceTypes, ModelStages
33
31
  from zenml.exceptions import EntityExistsError
34
32
  from zenml.logger import get_logger
@@ -527,14 +525,6 @@ class Model(BaseModel):
527
525
  data["suppress_class_validation_warnings"] = True
528
526
  return data
529
527
 
530
- def _validate_config_in_runtime(self) -> "ModelVersionResponse":
531
- """Validate that config doesn't conflict with runtime environment.
532
-
533
- Returns:
534
- The model version based on configuration.
535
- """
536
- return self._get_or_create_model_version()
537
-
538
528
  def _get_or_create_model(self) -> "ModelResponse":
539
529
  """This method should get or create a model from Model Control Plane.
540
530
 
@@ -678,16 +668,6 @@ class Model(BaseModel):
678
668
  if isinstance(self.version, str):
679
669
  self.version = format_name_template(self.version)
680
670
 
681
- zenml_client = Client()
682
- model_version_request = ModelVersionRequest(
683
- user=zenml_client.active_user.id,
684
- workspace=zenml_client.active_workspace.id,
685
- name=str(self.version) if self.version else None,
686
- description=self.description,
687
- model=model.id,
688
- tags=self.tags,
689
- )
690
- mv_request = ModelVersionRequest.model_validate(model_version_request)
691
671
  try:
692
672
  if self.version or self.model_version_id:
693
673
  model_version = self._get_model_version()
@@ -717,60 +697,34 @@ class Model(BaseModel):
717
697
  " as an example. You can explore model versions using "
718
698
  f"`zenml model version list -n {self.name}` CLI command."
719
699
  )
720
- retries_made = 0
721
- for i in range(MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION):
722
- try:
723
- model_version = (
724
- zenml_client.zen_store.create_model_version(
725
- model_version=mv_request
726
- )
727
- )
728
- break
729
- except EntityExistsError as e:
730
- if i == MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION - 1:
731
- raise RuntimeError(
732
- f"Failed to create model version "
733
- f"`{self.version if self.version else 'new'}` "
734
- f"in model `{self.name}`. Retried {retries_made} times. "
735
- "This could be driven by exceptionally high concurrency of "
736
- "pipeline runs. Please, reach out to us on ZenML Slack for support."
737
- ) from e
738
- # smoothed exponential back-off, it will go as 0.2, 0.3,
739
- # 0.45, 0.68, 1.01, 1.52, 2.28, 3.42, 5.13, 7.69, ...
740
- sleep = 0.2 * 1.5**i
741
- logger.debug(
742
- f"Failed to create new model version for "
743
- f"model `{self.name}`. Retrying in {sleep}..."
744
- )
745
- time.sleep(sleep)
746
- retries_made += 1
747
- self.version = model_version.name
700
+
701
+ client = Client()
702
+ model_version_request = ModelVersionRequest(
703
+ user=client.active_user.id,
704
+ workspace=client.active_workspace.id,
705
+ name=str(self.version) if self.version else None,
706
+ description=self.description,
707
+ model=model.id,
708
+ tags=self.tags,
709
+ )
710
+ model_version = client.zen_store.create_model_version(
711
+ model_version=model_version_request
712
+ )
713
+
748
714
  self._created_model_version = True
749
715
 
750
716
  logger.info(
751
717
  "Created new model version `%s` for model `%s`.",
752
- self.version,
718
+ model_version.name,
753
719
  self.name,
754
720
  )
755
721
 
722
+ self.version = model_version.name
756
723
  self.model_version_id = model_version.id
757
724
  self._model_id = model_version.model.id
758
725
  self._number = model_version.number
759
726
  return model_version
760
727
 
761
- def _merge(self, model: "Model") -> None:
762
- self.license = self.license or model.license
763
- self.description = self.description or model.description
764
- self.audience = self.audience or model.audience
765
- self.use_cases = self.use_cases or model.use_cases
766
- self.limitations = self.limitations or model.limitations
767
- self.trade_offs = self.trade_offs or model.trade_offs
768
- self.ethics = self.ethics or model.ethics
769
- if model.tags is not None:
770
- self.tags = list(
771
- {t for t in self.tags or []}.union(set(model.tags))
772
- )
773
-
774
728
  def __hash__(self) -> int:
775
729
  """Get hash of the `Model`.
776
730
 
@@ -71,7 +71,8 @@ of any potential costs:
71
71
 
72
72
  - An S3 bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/s3).
73
73
  - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
74
- - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker).
74
+ - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
75
+ as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
75
76
  - An IAM user and IAM role with the minimum necessary permissions to access the
76
77
  above resources.
77
78
  - An AWS access key used to give access to ZenML to connect to the above
@@ -257,13 +258,29 @@ console.
257
258
 
258
259
  config: Optional[str] = None
259
260
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
260
- config = f"""module "zenml_stack" {{
261
- source = "zenml-io/zenml-stack/aws"
261
+ config = f"""terraform {{
262
+ required_providers {{
263
+ aws = {{
264
+ source = "hashicorp/aws"
265
+ }}
266
+ zenml = {{
267
+ source = "zenml-io/zenml"
268
+ }}
269
+ }}
270
+ }}
262
271
 
272
+ provider "aws" {{
263
273
  region = "{self.location or "eu-central-1"}"
264
- zenml_server_url = "{self.zenml_server_url}"
265
- zenml_api_key = ""
266
- zenml_api_token = "{self.zenml_server_api_token}"
274
+ }}
275
+
276
+ provider "zenml" {{
277
+ server_url = "{self.zenml_server_url}"
278
+ api_token = "{self.zenml_server_api_token}"
279
+ }}
280
+
281
+ module "zenml_stack" {{
282
+ source = "zenml-io/zenml-stack/aws"
283
+
267
284
  zenml_stack_name = "{self.stack_name}"
268
285
  zenml_stack_deployment = "{self.deployment_type}"
269
286
  }}
@@ -259,14 +259,37 @@ ZenML's access to your Azure subscription.
259
259
  The configuration or script to deploy the ZenML stack to the
260
260
  specified cloud provider.
261
261
  """
262
- config = f"""module "zenml_stack" {{
262
+ config = f"""terraform {{
263
+ required_providers {{
264
+ azurerm = {{
265
+ source = "hashicorp/azurerm"
266
+ }}
267
+ azuread = {{
268
+ source = "hashicorp/azuread"
269
+ }}
270
+ zenml = {{
271
+ source = "zenml-io/zenml"
272
+ }}
273
+ }}
274
+ }}
275
+
276
+ provider "azurerm" {{
277
+ features {{
278
+ resource_group {{
279
+ prevent_deletion_if_contains_resources = false
280
+ }}
281
+ }}
282
+ }}
283
+
284
+ provider "zenml" {{
285
+ server_url = "{self.zenml_server_url}"
286
+ api_token = "{self.zenml_server_api_token}"
287
+ }}
288
+
289
+ module "zenml_stack" {{
263
290
  source = "zenml-io/zenml-stack/azure"
264
291
 
265
292
  location = "{self.location or "eastus"}"
266
- orchestrator = "azureml"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
293
  zenml_stack_name = "{self.stack_name}"
271
294
  zenml_stack_deployment = "{self.deployment_type}"
272
295
  }}
@@ -70,11 +70,12 @@ and are aware of any potential costs:
70
70
 
71
71
  - A GCS bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/gcp).
72
72
  - A Google Artifact Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/gcp).
73
- - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex).
73
+ - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
74
+ and as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/vertex).
74
75
  - GCP Cloud Build registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builders/gcp).
75
76
  - A GCP Service Account with the minimum necessary permissions to access the
76
77
  above resources.
77
- - An GCP Service Account access key used to give access to ZenML to connect to
78
+ - A GCP Service Account access key used to give access to ZenML to connect to
78
79
  the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/gcp-service-connector).
79
80
 
80
81
  The Deployment Manager deployment will automatically create a GCP Service
@@ -259,14 +260,30 @@ GCP project and to clean up the resources created by the stack by using
259
260
  )
260
261
 
261
262
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
262
- config = f"""module "zenml_stack" {{
263
+ config = f"""terraform {{
264
+ required_providers {{
265
+ google = {{
266
+ source = "hashicorp/google"
267
+ }}
268
+ zenml = {{
269
+ source = "zenml-io/zenml"
270
+ }}
271
+ }}
272
+ }}
273
+
274
+ provider "google" {{
275
+ region = "{self.location or "europe-west3"}"
276
+ project = your GCP project name
277
+ }}
278
+
279
+ provider "zenml" {{
280
+ server_url = "{self.zenml_server_url}"
281
+ api_token = "{self.zenml_server_api_token}"
282
+ }}
283
+
284
+ module "zenml_stack" {{
263
285
  source = "zenml-io/zenml-stack/gcp"
264
286
 
265
- project_id = "my-gcp-project"
266
- region = "{self.location or "europe-west3"}"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
287
  zenml_stack_name = "{self.stack_name}"
271
288
  zenml_stack_deployment = "{self.deployment_type}"
272
289
  }}
@@ -219,16 +219,14 @@ class ZenMLCloudStackDeployment(BaseModel):
219
219
  if stack.labels.get("zenml:deployment") != self.deployment_type:
220
220
  continue
221
221
 
222
- artifact_store = stack.components[
223
- StackComponentType.ARTIFACT_STORE
224
- ][0]
222
+ orchestrator = stack.components[StackComponentType.ORCHESTRATOR][0]
225
223
 
226
- if not artifact_store.connector:
224
+ if not orchestrator.connector:
227
225
  continue
228
226
 
229
227
  return DeployedStack(
230
228
  stack=stack,
231
- service_connector=artifact_store.connector,
229
+ service_connector=orchestrator.connector,
232
230
  )
233
231
 
234
232
  return None
@@ -23,6 +23,7 @@ from zenml.exceptions import (
23
23
  CredentialsNotValid,
24
24
  DoesNotExistException,
25
25
  DuplicateRunNameError,
26
+ EntityCreationError,
26
27
  EntityExistsError,
27
28
  IllegalOperationError,
28
29
  MethodNotAllowedError,
@@ -93,6 +94,7 @@ REST_API_EXCEPTIONS: List[Tuple[Type[Exception], int]] = [
93
94
  # 422 Unprocessable Entity
94
95
  (ValueError, 422),
95
96
  # 500 Internal Server Error
97
+ (EntityCreationError, 500),
96
98
  (RuntimeError, 500),
97
99
  # 501 Not Implemented,
98
100
  (NotImplementedError, 501),
@@ -0,0 +1,66 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Endpoint definitions for logs."""
15
+
16
+ from uuid import UUID
17
+
18
+ from fastapi import APIRouter, Security
19
+
20
+ from zenml.constants import (
21
+ API,
22
+ LOGS,
23
+ VERSION_1,
24
+ )
25
+ from zenml.models.v2.core.logs import LogsResponse
26
+ from zenml.zen_server.auth import AuthContext, authorize
27
+ from zenml.zen_server.exceptions import error_response
28
+ from zenml.zen_server.rbac.endpoint_utils import (
29
+ verify_permissions_and_get_entity,
30
+ )
31
+ from zenml.zen_server.utils import (
32
+ handle_exceptions,
33
+ zen_store,
34
+ )
35
+
36
+ router = APIRouter(
37
+ prefix=API + VERSION_1 + LOGS,
38
+ tags=["logs"],
39
+ responses={401: error_response, 403: error_response},
40
+ )
41
+
42
+
43
+ @router.get(
44
+ "/{logs_id}",
45
+ response_model=LogsResponse,
46
+ responses={401: error_response, 404: error_response, 422: error_response},
47
+ )
48
+ @handle_exceptions
49
+ def get_logs(
50
+ logs_id: UUID,
51
+ hydrate: bool = True,
52
+ _: AuthContext = Security(authorize),
53
+ ) -> LogsResponse:
54
+ """Returns the requested logs.
55
+
56
+ Args:
57
+ logs_id: ID of the logs.
58
+ hydrate: Flag deciding whether to hydrate the output model(s)
59
+ by including metadata fields in the response.
60
+
61
+ Returns:
62
+ The requested logs.
63
+ """
64
+ return verify_permissions_and_get_entity(
65
+ id=logs_id, get_method=zen_store().get_logs, hydrate=hydrate
66
+ )
@@ -64,6 +64,7 @@ from zenml.zen_server.routers import (
64
64
  devices_endpoints,
65
65
  event_source_endpoints,
66
66
  flavors_endpoints,
67
+ logs_endpoints,
67
68
  model_versions_endpoints,
68
69
  models_endpoints,
69
70
  pipeline_builds_endpoints,
@@ -414,6 +415,7 @@ app.include_router(code_repositories_endpoints.router)
414
415
  app.include_router(plugin_endpoints.plugin_router)
415
416
  app.include_router(event_source_endpoints.event_source_router)
416
417
  app.include_router(flavors_endpoints.router)
418
+ app.include_router(logs_endpoints.router)
417
419
  app.include_router(models_endpoints.router)
418
420
  app.include_router(model_versions_endpoints.router)
419
421
  app.include_router(model_versions_endpoints.model_version_artifacts_router)