zenml-nightly 0.58.2.dev20240618__py3-none-any.whl → 0.58.2.dev20240619__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. zenml/VERSION +1 -1
  2. zenml/_hub/client.py +8 -5
  3. zenml/actions/base_action.py +8 -10
  4. zenml/artifact_stores/base_artifact_store.py +20 -15
  5. zenml/artifact_stores/local_artifact_store.py +3 -2
  6. zenml/artifacts/artifact_config.py +34 -19
  7. zenml/artifacts/external_artifact.py +18 -8
  8. zenml/artifacts/external_artifact_config.py +14 -6
  9. zenml/artifacts/unmaterialized_artifact.py +2 -11
  10. zenml/cli/__init__.py +6 -0
  11. zenml/cli/artifact.py +20 -2
  12. zenml/cli/served_model.py +0 -1
  13. zenml/cli/server.py +3 -3
  14. zenml/cli/utils.py +36 -40
  15. zenml/cli/web_login.py +2 -2
  16. zenml/client.py +198 -24
  17. zenml/client_lazy_loader.py +20 -14
  18. zenml/config/base_settings.py +5 -6
  19. zenml/config/build_configuration.py +1 -1
  20. zenml/config/compiler.py +3 -3
  21. zenml/config/docker_settings.py +27 -28
  22. zenml/config/global_config.py +33 -37
  23. zenml/config/pipeline_configurations.py +8 -11
  24. zenml/config/pipeline_run_configuration.py +6 -2
  25. zenml/config/pipeline_spec.py +3 -4
  26. zenml/config/resource_settings.py +8 -9
  27. zenml/config/schedule.py +16 -20
  28. zenml/config/secret_reference_mixin.py +6 -3
  29. zenml/config/secrets_store_config.py +16 -23
  30. zenml/config/server_config.py +50 -46
  31. zenml/config/settings_resolver.py +1 -1
  32. zenml/config/source.py +45 -35
  33. zenml/config/step_configurations.py +53 -31
  34. zenml/config/store_config.py +20 -19
  35. zenml/config/strict_base_model.py +2 -6
  36. zenml/constants.py +26 -2
  37. zenml/container_registries/base_container_registry.py +3 -2
  38. zenml/container_registries/default_container_registry.py +3 -3
  39. zenml/event_hub/base_event_hub.py +1 -1
  40. zenml/event_sources/base_event_source.py +11 -16
  41. zenml/exceptions.py +4 -0
  42. zenml/integrations/airflow/__init__.py +2 -10
  43. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +6 -7
  44. zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +13 -249
  45. zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
  46. zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +5 -4
  47. zenml/integrations/aws/__init__.py +1 -1
  48. zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
  49. zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +11 -5
  50. zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
  51. zenml/integrations/aws/service_connectors/aws_service_connector.py +5 -4
  52. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +4 -4
  53. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -3
  54. zenml/integrations/azure/step_operators/azureml_step_operator.py +1 -1
  55. zenml/integrations/bentoml/steps/bentoml_deployer.py +1 -1
  56. zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +8 -12
  57. zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +1 -1
  58. zenml/integrations/evidently/__init__.py +3 -4
  59. zenml/integrations/evidently/column_mapping.py +11 -3
  60. zenml/integrations/evidently/data_validators/evidently_data_validator.py +21 -3
  61. zenml/integrations/evidently/metrics.py +5 -6
  62. zenml/integrations/evidently/tests.py +5 -6
  63. zenml/integrations/facets/models.py +2 -6
  64. zenml/integrations/feast/__init__.py +3 -1
  65. zenml/integrations/feast/feature_stores/feast_feature_store.py +0 -23
  66. zenml/integrations/gcp/__init__.py +1 -1
  67. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +1 -1
  68. zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +1 -1
  69. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +234 -103
  70. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +57 -42
  71. zenml/integrations/github/code_repositories/github_code_repository.py +1 -1
  72. zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +9 -13
  73. zenml/integrations/great_expectations/__init__.py +1 -1
  74. zenml/integrations/great_expectations/data_validators/ge_data_validator.py +44 -44
  75. zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
  76. zenml/integrations/great_expectations/ge_store_backend.py +24 -11
  77. zenml/integrations/great_expectations/materializers/ge_materializer.py +3 -3
  78. zenml/integrations/great_expectations/utils.py +5 -5
  79. zenml/integrations/huggingface/__init__.py +3 -0
  80. zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +1 -1
  81. zenml/integrations/huggingface/steps/__init__.py +3 -0
  82. zenml/integrations/huggingface/steps/accelerate_runner.py +149 -0
  83. zenml/integrations/huggingface/steps/huggingface_deployer.py +2 -2
  84. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +1 -1
  85. zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +4 -3
  86. zenml/integrations/kubeflow/__init__.py +1 -1
  87. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +48 -81
  88. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +295 -245
  89. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +1 -1
  90. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +11 -2
  91. zenml/integrations/kubernetes/pod_settings.py +17 -31
  92. zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +8 -7
  93. zenml/integrations/label_studio/__init__.py +1 -3
  94. zenml/integrations/label_studio/annotators/label_studio_annotator.py +3 -4
  95. zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +2 -2
  96. zenml/integrations/langchain/materializers/document_materializer.py +44 -8
  97. zenml/integrations/mlflow/__init__.py +9 -3
  98. zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +1 -1
  99. zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +29 -37
  100. zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +4 -4
  101. zenml/integrations/mlflow/steps/mlflow_deployer.py +1 -1
  102. zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
  103. zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +1 -1
  104. zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +9 -8
  105. zenml/integrations/seldon/seldon_client.py +52 -67
  106. zenml/integrations/seldon/services/seldon_deployment.py +3 -3
  107. zenml/integrations/seldon/steps/seldon_deployer.py +4 -4
  108. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +15 -5
  109. zenml/integrations/skypilot_aws/__init__.py +1 -1
  110. zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
  111. zenml/integrations/skypilot_azure/__init__.py +1 -1
  112. zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
  113. zenml/integrations/skypilot_gcp/__init__.py +2 -1
  114. zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
  115. zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +2 -2
  116. zenml/integrations/spark/flavors/spark_step_operator_flavor.py +1 -1
  117. zenml/integrations/tekton/__init__.py +1 -1
  118. zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
  119. zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +547 -233
  120. zenml/integrations/tensorboard/__init__.py +1 -12
  121. zenml/integrations/tensorboard/services/tensorboard_service.py +3 -5
  122. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +6 -6
  123. zenml/integrations/tensorflow/__init__.py +2 -10
  124. zenml/integrations/tensorflow/materializers/keras_materializer.py +17 -9
  125. zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +9 -14
  126. zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
  127. zenml/lineage_graph/lineage_graph.py +1 -1
  128. zenml/materializers/built_in_materializer.py +3 -3
  129. zenml/materializers/pydantic_materializer.py +2 -2
  130. zenml/metadata/lazy_load.py +4 -4
  131. zenml/metadata/metadata_types.py +64 -4
  132. zenml/model/model.py +79 -54
  133. zenml/model_deployers/base_model_deployer.py +14 -12
  134. zenml/model_registries/base_model_registry.py +17 -15
  135. zenml/models/__init__.py +79 -206
  136. zenml/models/v2/base/base.py +54 -41
  137. zenml/models/v2/base/base_plugin_flavor.py +2 -6
  138. zenml/models/v2/base/filter.py +91 -76
  139. zenml/models/v2/base/page.py +2 -12
  140. zenml/models/v2/base/scoped.py +4 -7
  141. zenml/models/v2/core/api_key.py +22 -8
  142. zenml/models/v2/core/artifact.py +2 -2
  143. zenml/models/v2/core/artifact_version.py +74 -40
  144. zenml/models/v2/core/code_repository.py +37 -10
  145. zenml/models/v2/core/component.py +65 -16
  146. zenml/models/v2/core/device.py +14 -4
  147. zenml/models/v2/core/event_source.py +1 -2
  148. zenml/models/v2/core/flavor.py +74 -8
  149. zenml/models/v2/core/logs.py +68 -8
  150. zenml/models/v2/core/model.py +8 -4
  151. zenml/models/v2/core/model_version.py +25 -6
  152. zenml/models/v2/core/model_version_artifact.py +51 -21
  153. zenml/models/v2/core/model_version_pipeline_run.py +45 -13
  154. zenml/models/v2/core/pipeline.py +37 -72
  155. zenml/models/v2/core/pipeline_build.py +29 -17
  156. zenml/models/v2/core/pipeline_deployment.py +18 -6
  157. zenml/models/v2/core/pipeline_namespace.py +113 -0
  158. zenml/models/v2/core/pipeline_run.py +50 -22
  159. zenml/models/v2/core/run_metadata.py +59 -36
  160. zenml/models/v2/core/schedule.py +37 -24
  161. zenml/models/v2/core/secret.py +31 -12
  162. zenml/models/v2/core/service.py +64 -36
  163. zenml/models/v2/core/service_account.py +24 -11
  164. zenml/models/v2/core/service_connector.py +219 -44
  165. zenml/models/v2/core/stack.py +45 -17
  166. zenml/models/v2/core/step_run.py +28 -8
  167. zenml/models/v2/core/tag.py +8 -4
  168. zenml/models/v2/core/trigger.py +2 -2
  169. zenml/models/v2/core/trigger_execution.py +1 -0
  170. zenml/models/v2/core/user.py +18 -21
  171. zenml/models/v2/core/workspace.py +13 -3
  172. zenml/models/v2/misc/build_item.py +3 -3
  173. zenml/models/v2/misc/external_user.py +2 -6
  174. zenml/models/v2/misc/hub_plugin_models.py +9 -9
  175. zenml/models/v2/misc/loaded_visualization.py +2 -2
  176. zenml/models/v2/misc/service_connector_type.py +8 -17
  177. zenml/models/v2/misc/user_auth.py +7 -2
  178. zenml/new/pipelines/build_utils.py +3 -3
  179. zenml/new/pipelines/pipeline.py +17 -13
  180. zenml/new/pipelines/run_utils.py +103 -1
  181. zenml/orchestrators/base_orchestrator.py +10 -7
  182. zenml/orchestrators/local_docker/local_docker_orchestrator.py +1 -1
  183. zenml/orchestrators/step_runner.py +3 -6
  184. zenml/orchestrators/utils.py +1 -1
  185. zenml/plugins/base_plugin_flavor.py +6 -10
  186. zenml/plugins/plugin_flavor_registry.py +3 -7
  187. zenml/secret/base_secret.py +7 -8
  188. zenml/service_connectors/docker_service_connector.py +4 -3
  189. zenml/service_connectors/service_connector.py +5 -12
  190. zenml/service_connectors/service_connector_registry.py +2 -4
  191. zenml/services/container/container_service.py +1 -1
  192. zenml/services/container/container_service_endpoint.py +1 -1
  193. zenml/services/local/local_service.py +1 -1
  194. zenml/services/local/local_service_endpoint.py +1 -1
  195. zenml/services/service.py +16 -10
  196. zenml/services/service_type.py +4 -5
  197. zenml/services/terraform/terraform_service.py +1 -1
  198. zenml/stack/flavor.py +1 -5
  199. zenml/stack/flavor_registry.py +4 -4
  200. zenml/stack/stack.py +4 -1
  201. zenml/stack/stack_component.py +55 -31
  202. zenml/steps/base_step.py +34 -28
  203. zenml/steps/entrypoint_function_utils.py +3 -5
  204. zenml/steps/utils.py +12 -14
  205. zenml/utils/cuda_utils.py +50 -0
  206. zenml/utils/deprecation_utils.py +18 -20
  207. zenml/utils/dict_utils.py +1 -1
  208. zenml/utils/filesync_model.py +65 -28
  209. zenml/utils/function_utils.py +260 -0
  210. zenml/utils/json_utils.py +131 -0
  211. zenml/utils/mlstacks_utils.py +2 -2
  212. zenml/utils/pydantic_utils.py +270 -62
  213. zenml/utils/secret_utils.py +65 -12
  214. zenml/utils/source_utils.py +2 -2
  215. zenml/utils/typed_model.py +5 -3
  216. zenml/utils/typing_utils.py +243 -0
  217. zenml/utils/yaml_utils.py +1 -1
  218. zenml/zen_server/auth.py +2 -2
  219. zenml/zen_server/cloud_utils.py +6 -6
  220. zenml/zen_server/deploy/base_provider.py +1 -1
  221. zenml/zen_server/deploy/deployment.py +6 -8
  222. zenml/zen_server/deploy/docker/docker_zen_server.py +3 -4
  223. zenml/zen_server/deploy/local/local_provider.py +0 -1
  224. zenml/zen_server/deploy/local/local_zen_server.py +6 -6
  225. zenml/zen_server/deploy/terraform/terraform_zen_server.py +4 -6
  226. zenml/zen_server/exceptions.py +4 -1
  227. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +1 -1
  228. zenml/zen_server/pipeline_deployment/utils.py +48 -68
  229. zenml/zen_server/rbac/models.py +2 -5
  230. zenml/zen_server/rbac/utils.py +11 -14
  231. zenml/zen_server/routers/auth_endpoints.py +2 -2
  232. zenml/zen_server/routers/pipeline_builds_endpoints.py +1 -1
  233. zenml/zen_server/routers/runs_endpoints.py +1 -1
  234. zenml/zen_server/routers/secrets_endpoints.py +3 -2
  235. zenml/zen_server/routers/server_endpoints.py +1 -1
  236. zenml/zen_server/routers/steps_endpoints.py +1 -1
  237. zenml/zen_server/routers/workspaces_endpoints.py +1 -1
  238. zenml/zen_stores/base_zen_store.py +46 -9
  239. zenml/zen_stores/migrations/utils.py +42 -46
  240. zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +1 -1
  241. zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +5 -3
  242. zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +1 -1
  243. zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +8 -10
  244. zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +3 -3
  245. zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +10 -12
  246. zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +3 -2
  247. zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +4 -4
  248. zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +3 -2
  249. zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +2 -2
  250. zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +3 -2
  251. zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +8 -7
  252. zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +6 -4
  253. zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +2 -2
  254. zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +1 -1
  255. zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +2 -2
  256. zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +4 -4
  257. zenml/zen_stores/migrations/versions/alembic_start.py +1 -1
  258. zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +4 -4
  259. zenml/zen_stores/rest_zen_store.py +109 -49
  260. zenml/zen_stores/schemas/api_key_schemas.py +1 -1
  261. zenml/zen_stores/schemas/artifact_schemas.py +8 -8
  262. zenml/zen_stores/schemas/artifact_visualization_schemas.py +3 -3
  263. zenml/zen_stores/schemas/code_repository_schemas.py +1 -1
  264. zenml/zen_stores/schemas/component_schemas.py +8 -3
  265. zenml/zen_stores/schemas/device_schemas.py +8 -6
  266. zenml/zen_stores/schemas/event_source_schemas.py +3 -4
  267. zenml/zen_stores/schemas/flavor_schemas.py +5 -3
  268. zenml/zen_stores/schemas/model_schemas.py +26 -1
  269. zenml/zen_stores/schemas/pipeline_build_schemas.py +1 -1
  270. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +4 -4
  271. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -6
  272. zenml/zen_stores/schemas/pipeline_schemas.py +5 -2
  273. zenml/zen_stores/schemas/run_metadata_schemas.py +2 -2
  274. zenml/zen_stores/schemas/secret_schemas.py +8 -5
  275. zenml/zen_stores/schemas/server_settings_schemas.py +3 -1
  276. zenml/zen_stores/schemas/service_connector_schemas.py +1 -1
  277. zenml/zen_stores/schemas/service_schemas.py +11 -2
  278. zenml/zen_stores/schemas/stack_schemas.py +1 -1
  279. zenml/zen_stores/schemas/step_run_schemas.py +11 -11
  280. zenml/zen_stores/schemas/tag_schemas.py +6 -2
  281. zenml/zen_stores/schemas/trigger_schemas.py +2 -2
  282. zenml/zen_stores/schemas/user_schemas.py +2 -2
  283. zenml/zen_stores/schemas/workspace_schemas.py +3 -1
  284. zenml/zen_stores/secrets_stores/aws_secrets_store.py +19 -20
  285. zenml/zen_stores/secrets_stores/azure_secrets_store.py +17 -20
  286. zenml/zen_stores/secrets_stores/base_secrets_store.py +79 -12
  287. zenml/zen_stores/secrets_stores/gcp_secrets_store.py +17 -20
  288. zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +4 -8
  289. zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +10 -7
  290. zenml/zen_stores/secrets_stores/sql_secrets_store.py +5 -6
  291. zenml/zen_stores/sql_zen_store.py +196 -120
  292. zenml/zen_stores/zen_store_interface.py +33 -0
  293. {zenml_nightly-0.58.2.dev20240618.dist-info → zenml_nightly-0.58.2.dev20240619.dist-info}/METADATA +8 -7
  294. {zenml_nightly-0.58.2.dev20240618.dist-info → zenml_nightly-0.58.2.dev20240619.dist-info}/RECORD +297 -294
  295. zenml/integrations/kubeflow/utils.py +0 -95
  296. zenml/models/v2/base/internal.py +0 -37
  297. zenml/models/v2/base/update.py +0 -44
  298. {zenml_nightly-0.58.2.dev20240618.dist-info → zenml_nightly-0.58.2.dev20240619.dist-info}/LICENSE +0 -0
  299. {zenml_nightly-0.58.2.dev20240618.dist-info → zenml_nightly-0.58.2.dev20240619.dist-info}/WHEEL +0 -0
  300. {zenml_nightly-0.58.2.dev20240618.dist-info → zenml_nightly-0.58.2.dev20240619.dist-info}/entry_points.txt +0 -0
zenml/cli/web_login.py CHANGED
@@ -101,7 +101,7 @@ def web_login(url: str, verify_ssl: Union[str, bool]) -> str:
101
101
  "Content-Type": "application/x-www-form-urlencoded",
102
102
  "User-Agent": user_agent_header.encode(),
103
103
  },
104
- data=auth_request.dict(),
104
+ data=auth_request.model_dump(),
105
105
  verify=verify_ssl,
106
106
  timeout=DEFAULT_HTTP_TIMEOUT,
107
107
  )
@@ -152,7 +152,7 @@ def web_login(url: str, verify_ssl: Union[str, bool]) -> str:
152
152
  response = requests.post(
153
153
  login_url,
154
154
  headers={"Content-Type": "application/x-www-form-urlencoded"},
155
- data=token_request.dict(),
155
+ data=token_request.model_dump(),
156
156
  verify=verify_ssl,
157
157
  timeout=DEFAULT_HTTP_TIMEOUT,
158
158
  )
zenml/client.py CHANGED
@@ -17,6 +17,7 @@ import functools
17
17
  import json
18
18
  import os
19
19
  from abc import ABCMeta
20
+ from collections import Counter
20
21
  from datetime import datetime
21
22
  from functools import partial
22
23
  from pathlib import Path
@@ -36,13 +37,14 @@ from typing import (
36
37
  )
37
38
  from uuid import UUID, uuid4
38
39
 
39
- from pydantic import SecretStr
40
+ from pydantic import ConfigDict, SecretStr
40
41
 
41
42
  from zenml.client_lazy_loader import (
42
43
  client_lazy_loader,
43
44
  evaluate_all_lazy_load_args_in_client_methods,
44
45
  )
45
46
  from zenml.config.global_config import GlobalConfiguration
47
+ from zenml.config.pipeline_run_configuration import PipelineRunConfiguration
46
48
  from zenml.config.source import Source
47
49
  from zenml.constants import (
48
50
  ENV_ZENML_ACTIVE_STACK_ID,
@@ -247,18 +249,14 @@ class ClientConfiguration(FileSyncModel):
247
249
  self.active_stack_id = stack.id
248
250
  self._active_stack = stack
249
251
 
250
- class Config:
251
- """Pydantic configuration class."""
252
-
252
+ model_config = ConfigDict(
253
253
  # Validate attributes when assigning them. We need to set this in order
254
254
  # to have a mix of mutable and immutable attributes
255
- validate_assignment = True
255
+ validate_assignment=True,
256
256
  # Allow extra attributes from configs of previous ZenML versions to
257
257
  # permit downgrading
258
- extra = "allow"
259
- # all attributes with leading underscore are private and therefore
260
- # are mutable and not included in serialization
261
- underscore_attrs_are_private = True
258
+ extra="allow",
259
+ )
262
260
 
263
261
 
264
262
  class ClientMetaClass(ABCMeta):
@@ -1304,7 +1302,7 @@ class Client(metaclass=ClientMetaClass):
1304
1302
  )
1305
1303
 
1306
1304
  # Create the update model
1307
- update_model = StackUpdate( # type: ignore[call-arg]
1305
+ update_model = StackUpdate(
1308
1306
  workspace=self.active_workspace.id,
1309
1307
  user=self.active_user.id,
1310
1308
  stack_spec_path=stack_spec_file,
@@ -1596,7 +1594,7 @@ class Client(metaclass=ClientMetaClass):
1596
1594
  service_request = ServiceRequest(
1597
1595
  name=config.service_name,
1598
1596
  service_type=service_type,
1599
- config=config.dict(),
1597
+ config=config.model_dump(),
1600
1598
  workspace=self.active_workspace.id,
1601
1599
  user=self.active_user.id,
1602
1600
  model_version_id=model_version_id,
@@ -2030,7 +2028,7 @@ class Client(metaclass=ClientMetaClass):
2030
2028
  allow_name_prefix_match=False,
2031
2029
  )
2032
2030
 
2033
- update_model = ComponentUpdate( # type: ignore[call-arg]
2031
+ update_model = ComponentUpdate(
2034
2032
  workspace=self.active_workspace.id,
2035
2033
  user=self.active_user.id,
2036
2034
  component_spec_path=component_spec_path,
@@ -2448,6 +2446,171 @@ class Client(metaclass=ClientMetaClass):
2448
2446
  )
2449
2447
  self.zen_store.delete_pipeline(pipeline_id=pipeline.id)
2450
2448
 
2449
+ @_fail_for_sql_zen_store
2450
+ def trigger_pipeline(
2451
+ self,
2452
+ pipeline_name_or_id: Union[str, UUID, None] = None,
2453
+ pipeline_version: Optional[str] = None,
2454
+ run_configuration: Optional[PipelineRunConfiguration] = None,
2455
+ config_path: Optional[str] = None,
2456
+ deployment_id: Optional[UUID] = None,
2457
+ build_id: Optional[UUID] = None,
2458
+ stack_name_or_id: Union[str, UUID, None] = None,
2459
+ synchronous: bool = False,
2460
+ ) -> PipelineRunResponse:
2461
+ """Trigger a pipeline from the server.
2462
+
2463
+ Usage examples:
2464
+ * Run the latest runnable build for the latest version of a pipeline:
2465
+ ```python
2466
+ Client().trigger_pipeline(pipeline_name_or_id=<NAME>)
2467
+ ```
2468
+ * Run the latest runnable build for a specific version of a pipeline:
2469
+ ```python
2470
+ Client().trigger_pipeline(
2471
+ pipeline_name_or_id=<NAME>,
2472
+ pipeline_version=<VERSION>
2473
+ )
2474
+ ```
2475
+ * Run a specific pipeline version on a specific stack:
2476
+ ```python
2477
+ Client().trigger_pipeline(
2478
+ pipeline_name_or_id=<ID>,
2479
+ stack_name_or_id=<ID>
2480
+ )
2481
+ ```
2482
+ * Run a specific deployment:
2483
+ ```python
2484
+ Client().trigger_pipeline(deployment_id=<ID>)
2485
+ ```
2486
+ * Run a specific build:
2487
+ ```python
2488
+ Client().trigger_pipeline(build_id=<ID>)
2489
+ ```
2490
+
2491
+ Args:
2492
+ pipeline_name_or_id: Name or ID of the pipeline. If not given,
2493
+ either the build or deployment that should be run needs to be
2494
+ specified.
2495
+ pipeline_version: Version of the pipeline. This is only used if a
2496
+ pipeline name is given.
2497
+ run_configuration: Configuration for the run. Either this or a
2498
+ path to a config file can be specified.
2499
+ config_path: Path to a YAML configuration file. This file will be
2500
+ parsed as a `PipelineRunConfiguration` object. Either this or
2501
+ the configuration in code can be specified.
2502
+ deployment_id: ID of the deployment to run. Either this or a build
2503
+ to run can be specified.
2504
+ build_id: ID of the build to run. Either this or a deployment to
2505
+ run can be specified.
2506
+ stack_name_or_id: Name or ID of the stack on which to run the
2507
+ pipeline. If not specified, this method will try to find a
2508
+ runnable build on any stack.
2509
+ synchronous: If `True`, this method will wait until the triggered
2510
+ run is finished.
2511
+
2512
+ Raises:
2513
+ RuntimeError: If triggering the pipeline failed.
2514
+
2515
+ Returns:
2516
+ Model of the pipeline run.
2517
+ """
2518
+ from zenml.new.pipelines.run_utils import (
2519
+ validate_run_config_is_runnable_from_server,
2520
+ validate_stack_is_runnable_from_server,
2521
+ wait_for_pipeline_run_to_finish,
2522
+ )
2523
+
2524
+ if Counter([build_id, deployment_id, pipeline_name_or_id])[None] != 2:
2525
+ raise RuntimeError(
2526
+ "You need to specify exactly one of pipeline, build or "
2527
+ "deployment to trigger."
2528
+ )
2529
+
2530
+ if run_configuration and config_path:
2531
+ raise RuntimeError(
2532
+ "Only config path or runtime configuration can be specified."
2533
+ )
2534
+
2535
+ if config_path:
2536
+ run_configuration = PipelineRunConfiguration.from_yaml(config_path)
2537
+
2538
+ if run_configuration:
2539
+ validate_run_config_is_runnable_from_server(run_configuration)
2540
+
2541
+ if deployment_id:
2542
+ if stack_name_or_id:
2543
+ logger.warning(
2544
+ "Deployment ID and stack specified, ignoring the stack and "
2545
+ "using stack from deployment instead."
2546
+ )
2547
+
2548
+ run = self.zen_store.run_deployment(
2549
+ deployment_id=deployment_id,
2550
+ run_configuration=run_configuration,
2551
+ )
2552
+ elif build_id:
2553
+ if stack_name_or_id:
2554
+ logger.warning(
2555
+ "Build ID and stack specified, ignoring the stack and "
2556
+ "using stack from build instead."
2557
+ )
2558
+
2559
+ run = self.zen_store.run_build(
2560
+ build_id=build_id, run_configuration=run_configuration
2561
+ )
2562
+ else:
2563
+ assert pipeline_name_or_id
2564
+ pipeline = self.get_pipeline(
2565
+ name_id_or_prefix=pipeline_name_or_id, version=pipeline_version
2566
+ )
2567
+
2568
+ stack = None
2569
+ if stack_name_or_id:
2570
+ stack = self.get_stack(
2571
+ stack_name_or_id, allow_name_prefix_match=False
2572
+ )
2573
+ validate_stack_is_runnable_from_server(
2574
+ zen_store=self.zen_store, stack=stack
2575
+ )
2576
+
2577
+ builds = depaginate(
2578
+ partial(
2579
+ self.list_builds,
2580
+ pipeline_id=pipeline.id,
2581
+ stack_id=stack.id if stack else None,
2582
+ )
2583
+ )
2584
+
2585
+ for build in builds:
2586
+ if not build.template_deployment_id:
2587
+ continue
2588
+
2589
+ if not build.stack:
2590
+ continue
2591
+
2592
+ try:
2593
+ validate_stack_is_runnable_from_server(
2594
+ zen_store=self.zen_store, stack=build.stack
2595
+ )
2596
+ except ValueError:
2597
+ continue
2598
+
2599
+ run = self.zen_store.run_build(
2600
+ build_id=build.id, run_configuration=run_configuration
2601
+ )
2602
+ break
2603
+ else:
2604
+ raise RuntimeError(
2605
+ "Unable to find a runnable build for the given stack and "
2606
+ "pipeline."
2607
+ )
2608
+
2609
+ if synchronous:
2610
+ run = wait_for_pipeline_run_to_finish(run_id=run.id)
2611
+
2612
+ return run
2613
+
2451
2614
  # -------------------------------- Builds ----------------------------------
2452
2615
 
2453
2616
  def get_build(
@@ -2989,13 +3152,13 @@ class Client(metaclass=ClientMetaClass):
2989
3152
 
2990
3153
  def get_deployment(
2991
3154
  self,
2992
- id_or_prefix: str,
3155
+ id_or_prefix: Union[str, UUID],
2993
3156
  hydrate: bool = True,
2994
3157
  ) -> PipelineDeploymentResponse:
2995
3158
  """Get a deployment by id or prefix.
2996
3159
 
2997
3160
  Args:
2998
- id_or_prefix: The id or id prefix of the build.
3161
+ id_or_prefix: The id or id prefix of the deployment.
2999
3162
  hydrate: Flag deciding whether to hydrate the output model(s)
3000
3163
  by including metadata fields in the response.
3001
3164
 
@@ -3011,10 +3174,12 @@ class Client(metaclass=ClientMetaClass):
3011
3174
 
3012
3175
  # First interpret as full UUID
3013
3176
  if is_valid_uuid(id_or_prefix):
3014
- return self.zen_store.get_deployment(
3015
- UUID(id_or_prefix),
3016
- hydrate=hydrate,
3177
+ id_ = (
3178
+ UUID(id_or_prefix)
3179
+ if isinstance(id_or_prefix, str)
3180
+ else id_or_prefix
3017
3181
  )
3182
+ return self.zen_store.get_deployment(id_, hydrate=hydrate)
3018
3183
 
3019
3184
  entity = self.list_deployments(
3020
3185
  id=f"startswith:{id_or_prefix}",
@@ -4270,7 +4435,7 @@ class Client(metaclass=ClientMetaClass):
4270
4435
  hydrate=True,
4271
4436
  )
4272
4437
 
4273
- secret_update = SecretUpdate(name=new_name or secret.name) # type: ignore[call-arg]
4438
+ secret_update = SecretUpdate(name=new_name or secret.name)
4274
4439
 
4275
4440
  if new_scope:
4276
4441
  secret_update.scope = new_scope
@@ -4589,7 +4754,7 @@ class Client(metaclass=ClientMetaClass):
4589
4754
  repo = self.get_code_repository(
4590
4755
  name_id_or_prefix=name_id_or_prefix, allow_name_prefix_match=False
4591
4756
  )
4592
- update = CodeRepositoryUpdate( # type: ignore[call-arg]
4757
+ update = CodeRepositoryUpdate(
4593
4758
  name=name, description=description, logo_url=logo_url
4594
4759
  )
4595
4760
  return self.zen_store.update_code_repository(
@@ -5099,8 +5264,6 @@ class Client(metaclass=ClientMetaClass):
5099
5264
  expires_at=expires_at,
5100
5265
  expires_skew_tolerance=expires_skew_tolerance,
5101
5266
  expiration_seconds=expiration_seconds,
5102
- user=self.active_user.id,
5103
- workspace=self.active_workspace.id,
5104
5267
  )
5105
5268
 
5106
5269
  # Validate and configure the resources
@@ -5143,21 +5306,32 @@ class Client(metaclass=ClientMetaClass):
5143
5306
 
5144
5307
  if verify:
5145
5308
  # Prefer to verify the connector config server-side if the
5146
- # implementation if available there, because it ensures
5309
+ # implementation, if available there, because it ensures
5147
5310
  # that the connector can be shared with other users or used
5148
5311
  # from other machines and because some auth methods rely on the
5149
5312
  # server-side authentication environment
5313
+
5314
+ # Convert the update model to a request model for validation
5315
+ connector_request_dict = connector_update.model_dump()
5316
+ connector_request_dict.update(
5317
+ user=self.active_user.id,
5318
+ workspace=self.active_workspace.id,
5319
+ )
5320
+ connector_request = ServiceConnectorRequest.model_validate(
5321
+ connector_request_dict
5322
+ )
5323
+
5150
5324
  if connector.remote:
5151
5325
  connector_resources = (
5152
5326
  self.zen_store.verify_service_connector_config(
5153
- connector_update,
5327
+ service_connector=connector_request,
5154
5328
  list_resources=list_resources,
5155
5329
  )
5156
5330
  )
5157
5331
  else:
5158
5332
  connector_instance = (
5159
5333
  service_connector_registry.instantiate_connector(
5160
- model=connector_update
5334
+ model=connector_request,
5161
5335
  )
5162
5336
  )
5163
5337
  connector_resources = connector_instance.verify(
@@ -16,7 +16,7 @@
16
16
  import contextlib
17
17
  from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type
18
18
 
19
- from pydantic import BaseModel, PrivateAttr
19
+ from pydantic import BaseModel
20
20
 
21
21
  if TYPE_CHECKING:
22
22
  from zenml.client import Client
@@ -35,8 +35,7 @@ class ClientLazyLoader(BaseModel):
35
35
 
36
36
  method_name: str
37
37
  call_chain: List[_CallStep] = []
38
-
39
- _exclude_next_call: bool = PrivateAttr(False)
38
+ exclude_next_call: bool = False
40
39
 
41
40
  def __getattr__(self, name: str) -> "ClientLazyLoader":
42
41
  """Get attribute not defined in ClientLazyLoader.
@@ -54,7 +53,7 @@ class ClientLazyLoader(BaseModel):
54
53
  if name != "__deepcopy__":
55
54
  self_.call_chain.append(_CallStep(attribute_name=name))
56
55
  else:
57
- self_._exclude_next_call = True
56
+ self_.exclude_next_call = True
58
57
  return self_
59
58
 
60
59
  def __call__(self, *args: Any, **kwargs: Any) -> "ClientLazyLoader":
@@ -68,11 +67,11 @@ class ClientLazyLoader(BaseModel):
68
67
  self
69
68
  """
70
69
  # workaround to protect from infinitely looping over in deepcopy called in invocations
71
- if not self._exclude_next_call:
70
+ if not self.exclude_next_call:
72
71
  self.call_chain.append(
73
72
  _CallStep(is_call=True, call_args=args, call_kwargs=kwargs)
74
73
  )
75
- self._exclude_next_call = False
74
+ self.exclude_next_call = False
76
75
  return self
77
76
 
78
77
  def __getitem__(self, item: Any) -> "ClientLazyLoader":
@@ -99,18 +98,25 @@ class ClientLazyLoader(BaseModel):
99
98
  self: "ClientLazyLoader", self_: Any, call_chain_: List[_CallStep]
100
99
  ) -> Any:
101
100
  next_step = call_chain_.pop(0)
102
- if next_step.is_call:
103
- self_ = self_(*next_step.call_args, **next_step.call_kwargs)
104
- elif next_step.selector:
105
- self_ = self_[next_step.selector]
106
- elif next_step.attribute_name:
107
- self_ = getattr(self_, next_step.attribute_name)
108
- else:
101
+ try:
102
+ if next_step.is_call:
103
+ self_ = self_(
104
+ *next_step.call_args, **next_step.call_kwargs
105
+ )
106
+ elif next_step.selector:
107
+ self_ = self_[next_step.selector]
108
+ elif next_step.attribute_name:
109
+ self_ = getattr(self_, next_step.attribute_name)
110
+ else:
111
+ raise ValueError(
112
+ "Invalid call chain. Reach out to the ZenML team."
113
+ )
114
+ except Exception as e:
109
115
  raise RuntimeError(
110
116
  f"Failed to evaluate lazy load chain `{self.method_name}` "
111
117
  f"+ `{self.call_chain}`. Reach out to the ZenML team via "
112
118
  "Slack or GitHub to check further."
113
- )
119
+ ) from e
114
120
  return self_
115
121
 
116
122
  self_ = getattr(Client(), self.method_name)
@@ -16,7 +16,7 @@
16
16
  from enum import IntFlag, auto
17
17
  from typing import Any, ClassVar, Dict, Union
18
18
 
19
- from pydantic import Extra
19
+ from pydantic import ConfigDict
20
20
 
21
21
  from zenml.config.secret_reference_mixin import SecretReferenceMixin
22
22
 
@@ -46,11 +46,10 @@ class BaseSettings(SecretReferenceMixin):
46
46
  ConfigurationLevel.PIPELINE | ConfigurationLevel.STEP
47
47
  )
48
48
 
49
- class Config:
50
- """Pydantic configuration class."""
51
-
49
+ model_config = ConfigDict(
52
50
  # public attributes are immutable
53
- allow_mutation = False
51
+ frozen=True,
54
52
  # allow extra attributes so this class can be used to parse dicts
55
53
  # of arbitrary subclasses
56
- extra = Extra.allow
54
+ extra="allow",
55
+ )
@@ -60,7 +60,7 @@ class BuildConfiguration(BaseModel):
60
60
  The checksum.
61
61
  """
62
62
  hash_ = hashlib.md5() # nosec
63
- hash_.update(self.settings.json().encode())
63
+ hash_.update(self.settings.model_dump_json().encode())
64
64
  if self.entrypoint:
65
65
  hash_.update(self.entrypoint.encode())
66
66
 
zenml/config/compiler.py CHANGED
@@ -286,9 +286,9 @@ class Compiler:
286
286
  """
287
287
  assert stack_component.settings_class
288
288
  # Exclude additional config attributes that aren't part of the settings
289
- field_names = set(stack_component.settings_class.__fields__)
290
- default_settings = stack_component.settings_class.parse_obj(
291
- stack_component.config.dict(
289
+ field_names = set(stack_component.settings_class.model_fields.keys())
290
+ default_settings = stack_component.settings_class.model_validate(
291
+ stack_component.config.model_dump(
292
292
  include=field_names, exclude_unset=True, exclude_defaults=True
293
293
  )
294
294
  )
@@ -16,11 +16,13 @@
16
16
  from enum import Enum
17
17
  from typing import Any, Dict, List, Optional, Union
18
18
 
19
- from pydantic import Extra, root_validator
19
+ from pydantic import Field, model_validator
20
+ from pydantic_settings import SettingsConfigDict
20
21
 
21
22
  from zenml.config.base_settings import BaseSettings
22
23
  from zenml.logger import get_logger
23
24
  from zenml.utils import deprecation_utils
25
+ from zenml.utils.pydantic_utils import before_validator_handler
24
26
 
25
27
  logger = get_logger(__name__)
26
28
 
@@ -70,7 +72,7 @@ class DockerSettings(BaseSettings):
70
72
  --------------
71
73
  * No `dockerfile` specified: If any of the options regarding
72
74
  requirements, environment variables or copying files require us to build an
73
- image, ZenML will build this image. Otherwise the `parent_image` will be
75
+ image, ZenML will build this image. Otherwise, the `parent_image` will be
74
76
  used to run the pipeline.
75
77
  * `dockerfile` specified: ZenML will first build an image based on the
76
78
  specified Dockerfile. If any of the options regarding
@@ -190,8 +192,10 @@ class DockerSettings(BaseSettings):
190
192
  python_package_installer_args: Dict[str, Any] = {}
191
193
  replicate_local_python_environment: Optional[
192
194
  Union[List[str], PythonEnvironmentExportMethod]
193
- ] = None
194
- requirements: Union[None, str, List[str]] = None
195
+ ] = Field(default=None, union_mode="left_to_right")
196
+ requirements: Union[None, str, List[str]] = Field(
197
+ default=None, union_mode="left_to_right"
198
+ )
195
199
  required_integrations: List[str] = []
196
200
  required_hub_plugins: List[str] = []
197
201
  install_stack_requirements: bool = True
@@ -208,41 +212,40 @@ class DockerSettings(BaseSettings):
208
212
  "copy_files", "copy_global_config"
209
213
  )
210
214
 
211
- @root_validator(pre=True)
212
- def _migrate_copy_files(cls, values: Dict[str, Any]) -> Dict[str, Any]:
215
+ @model_validator(mode="before")
216
+ @classmethod
217
+ @before_validator_handler
218
+ def _migrate_copy_files(cls, data: Dict[str, Any]) -> Dict[str, Any]:
213
219
  """Migrates the value from the old copy_files attribute.
214
220
 
215
221
  Args:
216
- values: The settings values.
222
+ data: The settings values.
217
223
 
218
224
  Returns:
219
225
  The migrated settings values.
220
226
  """
221
- copy_files = values.get("copy_files", None)
227
+ copy_files = data.get("copy_files", None)
222
228
 
223
229
  if copy_files is None:
224
- return values
230
+ return data
225
231
 
226
- if values.get("source_files", None):
232
+ if data.get("source_files", None):
227
233
  # Ignore the copy files value in favor of the new source files
228
234
  logger.warning(
229
235
  "Both `copy_files` and `source_files` specified for the "
230
236
  "DockerSettings, ignoring the `copy_files` value."
231
237
  )
232
238
  elif copy_files is True:
233
- values["source_files"] = SourceFileMode.INCLUDE
239
+ data["source_files"] = SourceFileMode.INCLUDE
234
240
  elif copy_files is False:
235
- values["source_files"] = SourceFileMode.IGNORE
241
+ data["source_files"] = SourceFileMode.IGNORE
236
242
 
237
- return values
243
+ return data
238
244
 
239
- @root_validator
240
- def _validate_skip_build(cls, values: Dict[str, Any]) -> Dict[str, Any]:
245
+ @model_validator(mode="after")
246
+ def _validate_skip_build(self) -> "DockerSettings":
241
247
  """Ensures that a parent image is passed when trying to skip the build.
242
248
 
243
- Args:
244
- values: The settings values.
245
-
246
249
  Returns:
247
250
  The validated settings values.
248
251
 
@@ -250,10 +253,7 @@ class DockerSettings(BaseSettings):
250
253
  ValueError: If the build should be skipped but no parent image
251
254
  was specified.
252
255
  """
253
- skip_build = values.get("skip_build", False)
254
- parent_image = values.get("parent_image")
255
-
256
- if skip_build and not parent_image:
256
+ if self.skip_build and not self.parent_image:
257
257
  raise ValueError(
258
258
  "Docker settings that specify `skip_build=True` must always "
259
259
  "contain a `parent_image`. This parent image will be used "
@@ -261,12 +261,11 @@ class DockerSettings(BaseSettings):
261
261
  "Docker builds on top of it."
262
262
  )
263
263
 
264
- return values
265
-
266
- class Config:
267
- """Pydantic configuration class."""
264
+ return self
268
265
 
266
+ model_config = SettingsConfigDict(
269
267
  # public attributes are immutable
270
- allow_mutation = False
268
+ frozen=True,
271
269
  # prevent extra attributes during model initialization
272
- extra = Extra.forbid
270
+ extra="forbid",
271
+ )