zenml-nightly 0.55.0.dev20240124__py3-none-any.whl → 0.72.0.dev20250116__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1240) hide show
  1. zenml/VERSION +1 -1
  2. zenml/__init__.py +9 -12
  3. zenml/actions/__init__.py +14 -0
  4. zenml/actions/base_action.py +682 -0
  5. zenml/actions/pipeline_run/__init__.py +0 -0
  6. zenml/actions/pipeline_run/pipeline_run_action.py +223 -0
  7. zenml/analytics/context.py +70 -15
  8. zenml/analytics/enums.py +21 -17
  9. zenml/analytics/models.py +1 -0
  10. zenml/analytics/utils.py +19 -7
  11. zenml/annotators/base_annotator.py +4 -3
  12. zenml/artifact_stores/base_artifact_store.py +155 -72
  13. zenml/artifact_stores/local_artifact_store.py +3 -2
  14. zenml/artifacts/artifact_config.py +67 -58
  15. zenml/artifacts/external_artifact.py +24 -39
  16. zenml/artifacts/external_artifact_config.py +27 -37
  17. zenml/artifacts/preexisting_data_materializer.py +90 -0
  18. zenml/artifacts/unmaterialized_artifact.py +2 -12
  19. zenml/artifacts/utils.py +521 -220
  20. zenml/cli/__init__.py +1511 -456
  21. zenml/cli/annotator.py +48 -14
  22. zenml/cli/artifact.py +22 -4
  23. zenml/cli/authorized_device.py +1 -0
  24. zenml/cli/base.py +185 -22
  25. zenml/cli/code_repository.py +1 -0
  26. zenml/cli/formatter.py +1 -1
  27. zenml/cli/integration.py +119 -24
  28. zenml/cli/login.py +1053 -0
  29. zenml/cli/model.py +42 -15
  30. zenml/cli/model_registry.py +1 -2
  31. zenml/cli/pipeline.py +168 -82
  32. zenml/cli/secret.py +1 -2
  33. zenml/cli/served_model.py +56 -26
  34. zenml/cli/server.py +486 -619
  35. zenml/cli/service_accounts.py +55 -14
  36. zenml/cli/service_connectors.py +97 -6
  37. zenml/cli/stack.py +821 -531
  38. zenml/cli/stack_components.py +15 -598
  39. zenml/cli/tag.py +1 -0
  40. zenml/cli/text_utils.py +36 -2
  41. zenml/cli/user_management.py +204 -6
  42. zenml/cli/utils.py +411 -290
  43. zenml/client.py +1742 -298
  44. zenml/client_lazy_loader.py +224 -0
  45. zenml/code_repositories/base_code_repository.py +5 -4
  46. zenml/code_repositories/git/local_git_repository_context.py +1 -0
  47. zenml/code_repositories/local_repository_context.py +1 -0
  48. zenml/config/__init__.py +2 -0
  49. zenml/config/base_settings.py +6 -6
  50. zenml/config/build_configuration.py +43 -17
  51. zenml/config/compiler.py +82 -49
  52. zenml/config/docker_settings.py +139 -83
  53. zenml/config/global_config.py +260 -234
  54. zenml/config/pipeline_configurations.py +32 -11
  55. zenml/config/pipeline_run_configuration.py +15 -3
  56. zenml/config/pipeline_spec.py +6 -6
  57. zenml/config/resource_settings.py +8 -9
  58. zenml/config/retry_config.py +27 -0
  59. zenml/config/schedule.py +27 -18
  60. zenml/config/secret_reference_mixin.py +8 -4
  61. zenml/config/secrets_store_config.py +16 -24
  62. zenml/config/server_config.py +434 -51
  63. zenml/config/settings_resolver.py +2 -1
  64. zenml/config/source.py +97 -31
  65. zenml/config/step_configurations.py +83 -39
  66. zenml/config/step_run_info.py +3 -0
  67. zenml/config/store_config.py +20 -54
  68. zenml/config/strict_base_model.py +2 -6
  69. zenml/console.py +0 -1
  70. zenml/constants.py +232 -74
  71. zenml/container_registries/azure_container_registry.py +1 -0
  72. zenml/container_registries/base_container_registry.py +7 -3
  73. zenml/container_registries/default_container_registry.py +4 -3
  74. zenml/container_registries/dockerhub_container_registry.py +1 -0
  75. zenml/container_registries/gcp_container_registry.py +1 -0
  76. zenml/container_registries/github_container_registry.py +2 -10
  77. zenml/data_validators/base_data_validator.py +2 -2
  78. zenml/entrypoints/base_entrypoint_configuration.py +76 -17
  79. zenml/entrypoints/pipeline_entrypoint_configuration.py +1 -0
  80. zenml/entrypoints/step_entrypoint_configuration.py +21 -2
  81. zenml/enums.py +91 -9
  82. zenml/environment.py +52 -319
  83. zenml/{steps/base_parameters.py → event_hub/__init__.py} +5 -7
  84. zenml/event_hub/base_event_hub.py +196 -0
  85. zenml/event_hub/event_hub.py +181 -0
  86. zenml/event_sources/__init__.py +14 -0
  87. zenml/{lineage_graph/edge.py → event_sources/base_event.py} +5 -7
  88. zenml/event_sources/base_event_source.py +695 -0
  89. zenml/event_sources/webhooks/__init__.py +14 -0
  90. zenml/event_sources/webhooks/base_webhook_event_source.py +231 -0
  91. zenml/exceptions.py +40 -41
  92. zenml/feature_stores/base_feature_store.py +4 -6
  93. zenml/hooks/hook_validators.py +3 -11
  94. zenml/image_builders/base_image_builder.py +5 -2
  95. zenml/image_builders/build_context.py +24 -80
  96. zenml/image_builders/local_image_builder.py +14 -6
  97. zenml/integrations/__init__.py +16 -4
  98. zenml/integrations/airflow/__init__.py +3 -5
  99. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +15 -7
  100. zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +15 -252
  101. zenml/integrations/airflow/orchestrators/dag_generator.py +5 -3
  102. zenml/integrations/argilla/__init__.py +46 -0
  103. zenml/integrations/argilla/annotators/__init__.py +20 -0
  104. zenml/integrations/argilla/annotators/argilla_annotator.py +443 -0
  105. zenml/{post_execution → integrations/argilla/flavors}/__init__.py +9 -13
  106. zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +150 -0
  107. zenml/integrations/aws/__init__.py +7 -3
  108. zenml/integrations/aws/container_registries/aws_container_registry.py +44 -8
  109. zenml/integrations/aws/flavors/__init__.py +6 -0
  110. zenml/integrations/aws/flavors/aws_container_registry_flavor.py +3 -2
  111. zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
  112. zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +104 -16
  113. zenml/integrations/aws/flavors/sagemaker_step_operator_flavor.py +6 -2
  114. zenml/integrations/aws/image_builders/__init__.py +20 -0
  115. zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
  116. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +421 -133
  117. zenml/integrations/aws/service_connectors/aws_service_connector.py +175 -48
  118. zenml/integrations/aws/step_operators/sagemaker_step_operator.py +1 -1
  119. zenml/integrations/azure/__init__.py +14 -5
  120. zenml/integrations/azure/azureml_utils.py +201 -0
  121. zenml/integrations/azure/flavors/__init__.py +11 -0
  122. zenml/integrations/azure/flavors/azureml.py +139 -0
  123. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +166 -0
  124. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +71 -18
  125. zenml/integrations/azure/orchestrators/__init__.py +19 -0
  126. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +583 -0
  127. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +82 -0
  128. zenml/integrations/azure/service_connectors/azure_service_connector.py +15 -6
  129. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -173
  130. zenml/integrations/bentoml/__init__.py +1 -1
  131. zenml/integrations/bentoml/constants.py +1 -1
  132. zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
  133. zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +128 -239
  134. zenml/integrations/bentoml/services/__init__.py +15 -4
  135. zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
  136. zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +85 -43
  137. zenml/integrations/bentoml/services/deployment_type.py +23 -0
  138. zenml/integrations/bentoml/steps/bento_builder.py +2 -0
  139. zenml/integrations/bentoml/steps/bentoml_deployer.py +97 -47
  140. zenml/integrations/bitbucket/__init__.py +42 -0
  141. zenml/integrations/bitbucket/plugins/__init__.py +20 -0
  142. zenml/integrations/bitbucket/plugins/bitbucket_webhook_event_source_flavor.py +43 -0
  143. zenml/integrations/bitbucket/plugins/event_sources/__init__.py +0 -0
  144. zenml/integrations/bitbucket/plugins/event_sources/bitbucket_webhook_event_source.py +486 -0
  145. zenml/integrations/comet/__init__.py +49 -0
  146. zenml/integrations/{kserve/services → comet/experiment_trackers}/__init__.py +5 -6
  147. zenml/integrations/comet/experiment_trackers/comet_experiment_tracker.py +175 -0
  148. zenml/integrations/comet/flavors/__init__.py +24 -0
  149. zenml/integrations/comet/flavors/comet_experiment_tracker_flavor.py +129 -0
  150. zenml/integrations/constants.py +14 -2
  151. zenml/integrations/databricks/__init__.py +70 -0
  152. zenml/{lineage_graph → integrations/databricks/flavors}/__init__.py +14 -18
  153. zenml/integrations/databricks/flavors/databricks_model_deployer_flavor.py +118 -0
  154. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +174 -0
  155. zenml/integrations/{kserve → databricks}/model_deployers/__init__.py +5 -5
  156. zenml/integrations/databricks/model_deployers/databricks_model_deployer.py +249 -0
  157. zenml/integrations/databricks/orchestrators/__init__.py +20 -0
  158. zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +501 -0
  159. zenml/integrations/databricks/orchestrators/databricks_orchestrator_entrypoint_config.py +97 -0
  160. zenml/integrations/databricks/services/__init__.py +19 -0
  161. zenml/integrations/databricks/services/databricks_deployment.py +407 -0
  162. zenml/integrations/{gcp/orchestrators/vertex_scheduler → databricks/utils}/__init__.py +2 -2
  163. zenml/integrations/databricks/utils/databricks_utils.py +87 -0
  164. zenml/integrations/deepchecks/__init__.py +30 -10
  165. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +59 -18
  166. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  167. zenml/integrations/deepchecks/materializers/deepchecks_results_materializer.py +3 -3
  168. zenml/integrations/deepchecks/validation_checks.py +62 -35
  169. zenml/integrations/discord/__init__.py +1 -0
  170. zenml/integrations/discord/steps/discord_alerter_ask_step.py +1 -0
  171. zenml/integrations/discord/steps/discord_alerter_post_step.py +1 -0
  172. zenml/integrations/evidently/__init__.py +22 -4
  173. zenml/integrations/evidently/column_mapping.py +11 -3
  174. zenml/integrations/evidently/data_validators/evidently_data_validator.py +24 -6
  175. zenml/integrations/evidently/metrics.py +7 -8
  176. zenml/integrations/evidently/tests.py +7 -8
  177. zenml/integrations/facets/__init__.py +22 -5
  178. zenml/integrations/facets/models.py +2 -7
  179. zenml/integrations/feast/__init__.py +19 -3
  180. zenml/integrations/feast/feature_stores/feast_feature_store.py +13 -32
  181. zenml/integrations/gcp/__init__.py +6 -6
  182. zenml/integrations/gcp/artifact_stores/gcp_artifact_store.py +6 -0
  183. zenml/integrations/gcp/flavors/gcp_artifact_store_flavor.py +1 -0
  184. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +24 -2
  185. zenml/integrations/gcp/flavors/vertex_step_operator_flavor.py +14 -1
  186. zenml/integrations/gcp/google_credentials_mixin.py +1 -1
  187. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +519 -296
  188. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +788 -113
  189. zenml/integrations/gcp/step_operators/vertex_step_operator.py +9 -1
  190. zenml/integrations/github/__init__.py +15 -0
  191. zenml/integrations/github/code_repositories/github_code_repository.py +3 -2
  192. zenml/integrations/github/plugins/__init__.py +19 -0
  193. zenml/integrations/github/plugins/event_sources/__init__.py +0 -0
  194. zenml/integrations/github/plugins/event_sources/github_webhook_event_source.py +510 -0
  195. zenml/integrations/github/plugins/github_webhook_event_source_flavor.py +43 -0
  196. zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +7 -1
  197. zenml/integrations/great_expectations/__init__.py +22 -10
  198. zenml/integrations/great_expectations/data_validators/ge_data_validator.py +61 -57
  199. zenml/integrations/great_expectations/flavors/great_expectations_data_validator_flavor.py +35 -2
  200. zenml/integrations/great_expectations/ge_store_backend.py +24 -11
  201. zenml/integrations/great_expectations/materializers/ge_materializer.py +9 -9
  202. zenml/integrations/great_expectations/utils.py +5 -5
  203. zenml/integrations/huggingface/__init__.py +52 -1
  204. zenml/integrations/huggingface/flavors/__init__.py +26 -0
  205. zenml/integrations/huggingface/flavors/huggingface_model_deployer_flavor.py +130 -0
  206. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  207. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +102 -19
  208. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +18 -19
  209. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +104 -0
  210. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +18 -19
  211. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +13 -15
  212. zenml/integrations/huggingface/model_deployers/__init__.py +20 -0
  213. zenml/integrations/huggingface/model_deployers/huggingface_model_deployer.py +247 -0
  214. zenml/integrations/huggingface/services/__init__.py +19 -0
  215. zenml/integrations/huggingface/services/huggingface_deployment.py +292 -0
  216. zenml/integrations/huggingface/steps/__init__.py +21 -0
  217. zenml/integrations/huggingface/steps/accelerate_runner.py +166 -0
  218. zenml/integrations/huggingface/steps/huggingface_deployer.py +110 -0
  219. zenml/integrations/hyperai/__init__.py +53 -0
  220. zenml/integrations/hyperai/flavors/__init__.py +20 -0
  221. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +170 -0
  222. zenml/integrations/hyperai/orchestrators/__init__.py +21 -0
  223. zenml/integrations/hyperai/orchestrators/hyperai_orchestrator.py +510 -0
  224. zenml/integrations/hyperai/service_connectors/__init__.py +20 -0
  225. zenml/integrations/hyperai/service_connectors/hyperai_service_connector.py +375 -0
  226. zenml/integrations/integration.py +101 -47
  227. zenml/integrations/kaniko/flavors/kaniko_image_builder_flavor.py +6 -44
  228. zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +9 -6
  229. zenml/integrations/kubeflow/__init__.py +4 -1
  230. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +67 -82
  231. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +306 -248
  232. zenml/integrations/kubernetes/__init__.py +6 -3
  233. zenml/integrations/kubernetes/flavors/__init__.py +8 -0
  234. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +43 -2
  235. zenml/integrations/kubernetes/flavors/kubernetes_step_operator_flavor.py +166 -0
  236. zenml/integrations/kubernetes/orchestrators/kube_utils.py +67 -12
  237. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +89 -21
  238. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator_entrypoint.py +39 -7
  239. zenml/integrations/kubernetes/orchestrators/manifest_utils.py +42 -19
  240. zenml/integrations/kubernetes/pod_settings.py +21 -31
  241. zenml/integrations/kubernetes/serialization_utils.py +3 -3
  242. zenml/integrations/kubernetes/service_connectors/kubernetes_service_connector.py +14 -11
  243. zenml/integrations/kubernetes/step_operators/__init__.py +22 -0
  244. zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +236 -0
  245. zenml/integrations/label_studio/__init__.py +1 -3
  246. zenml/integrations/label_studio/annotators/label_studio_annotator.py +46 -14
  247. zenml/integrations/label_studio/flavors/__init__.py +2 -0
  248. zenml/integrations/label_studio/flavors/label_studio_annotator_flavor.py +20 -5
  249. zenml/integrations/langchain/__init__.py +7 -1
  250. zenml/integrations/langchain/materializers/document_materializer.py +44 -8
  251. zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
  252. zenml/integrations/lightgbm/__init__.py +1 -0
  253. zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
  254. zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
  255. zenml/integrations/lightning/__init__.py +48 -0
  256. zenml/integrations/lightning/flavors/__init__.py +23 -0
  257. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +168 -0
  258. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  259. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +617 -0
  260. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +303 -0
  261. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  262. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  263. zenml/integrations/mlflow/__init__.py +55 -8
  264. zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +31 -13
  265. zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +32 -37
  266. zenml/integrations/mlflow/model_deployers/mlflow_model_deployer.py +42 -233
  267. zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +42 -49
  268. zenml/integrations/mlflow/services/mlflow_deployment.py +30 -5
  269. zenml/integrations/mlflow/steps/mlflow_deployer.py +25 -27
  270. zenml/integrations/mlflow/steps/mlflow_registry.py +3 -1
  271. zenml/integrations/modal/__init__.py +46 -0
  272. zenml/integrations/modal/flavors/__init__.py +26 -0
  273. zenml/integrations/modal/flavors/modal_step_operator_flavor.py +125 -0
  274. zenml/integrations/modal/step_operators/__init__.py +22 -0
  275. zenml/integrations/modal/step_operators/modal_step_operator.py +242 -0
  276. zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
  277. zenml/integrations/neptune/experiment_trackers/run_state.py +71 -55
  278. zenml/integrations/neptune/flavors/neptune_experiment_tracker_flavor.py +1 -1
  279. zenml/integrations/neural_prophet/__init__.py +6 -1
  280. zenml/integrations/numpy/__init__.py +32 -0
  281. zenml/integrations/{kserve/constants.py → numpy/materializers/__init__.py} +5 -4
  282. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  283. zenml/integrations/openai/__init__.py +1 -1
  284. zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
  285. zenml/{steps/external_artifact.py → integrations/pandas/__init__.py} +17 -11
  286. zenml/integrations/{kserve/custom_deployer → pandas/materializers}/__init__.py +5 -5
  287. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  288. zenml/integrations/pigeon/__init__.py +44 -0
  289. zenml/integrations/pigeon/annotators/__init__.py +20 -0
  290. zenml/integrations/pigeon/annotators/pigeon_annotator.py +330 -0
  291. zenml/integrations/{kserve → pigeon}/flavors/__init__.py +7 -7
  292. zenml/integrations/pigeon/flavors/pigeon_annotator_flavor.py +104 -0
  293. zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
  294. zenml/integrations/polars/__init__.py +1 -0
  295. zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
  296. zenml/integrations/prodigy/__init__.py +48 -0
  297. zenml/integrations/prodigy/annotators/__init__.py +20 -0
  298. zenml/integrations/prodigy/annotators/prodigy_annotator.py +275 -0
  299. zenml/integrations/prodigy/flavors/__init__.py +24 -0
  300. zenml/integrations/prodigy/flavors/prodigy_annotator_flavor.py +101 -0
  301. zenml/integrations/pycaret/__init__.py +6 -0
  302. zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
  303. zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
  304. zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
  305. zenml/integrations/registry.py +38 -1
  306. zenml/integrations/s3/__init__.py +2 -3
  307. zenml/integrations/s3/artifact_stores/s3_artifact_store.py +191 -9
  308. zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +22 -45
  309. zenml/integrations/s3/utils.py +39 -0
  310. zenml/integrations/seldon/__init__.py +18 -2
  311. zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +19 -141
  312. zenml/integrations/seldon/secret_schemas/secret_schemas.py +2 -2
  313. zenml/integrations/seldon/seldon_client.py +55 -70
  314. zenml/integrations/seldon/services/seldon_deployment.py +4 -5
  315. zenml/integrations/seldon/steps/seldon_deployer.py +21 -20
  316. zenml/integrations/sklearn/__init__.py +1 -1
  317. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +29 -7
  318. zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +47 -28
  319. zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
  320. zenml/integrations/skypilot_aws/__init__.py +3 -2
  321. zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py +1 -1
  322. zenml/integrations/skypilot_azure/__init__.py +2 -4
  323. zenml/integrations/skypilot_azure/flavors/skypilot_orchestrator_azure_vm_flavor.py +1 -1
  324. zenml/integrations/skypilot_gcp/__init__.py +3 -2
  325. zenml/integrations/skypilot_gcp/flavors/skypilot_orchestrator_gcp_vm_flavor.py +1 -1
  326. zenml/integrations/skypilot_kubernetes/__init__.py +52 -0
  327. zenml/integrations/skypilot_kubernetes/flavors/__init__.py +26 -0
  328. zenml/integrations/skypilot_kubernetes/flavors/skypilot_orchestrator_kubernetes_vm_flavor.py +125 -0
  329. zenml/integrations/skypilot_kubernetes/orchestrators/__init__.py +25 -0
  330. zenml/integrations/skypilot_kubernetes/orchestrators/skypilot_kubernetes_vm_orchestrator.py +74 -0
  331. zenml/integrations/skypilot_lambda/__init__.py +50 -0
  332. zenml/integrations/{kserve/secret_schemas → skypilot_lambda/flavors}/__init__.py +9 -12
  333. zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +130 -0
  334. zenml/{lineage_graph/node → integrations/skypilot_lambda/orchestrators}/__init__.py +8 -15
  335. zenml/integrations/skypilot_lambda/orchestrators/skypilot_lambda_vm_orchestrator.py +92 -0
  336. zenml/integrations/slack/__init__.py +1 -0
  337. zenml/integrations/slack/alerters/slack_alerter.py +22 -2
  338. zenml/integrations/slack/flavors/__init__.py +2 -0
  339. zenml/integrations/slack/flavors/slack_alerter_flavor.py +13 -4
  340. zenml/integrations/slack/steps/slack_alerter_ask_step.py +1 -0
  341. zenml/integrations/slack/steps/slack_alerter_post_step.py +1 -0
  342. zenml/integrations/spark/flavors/spark_step_operator_flavor.py +2 -39
  343. zenml/integrations/spark/step_operators/kubernetes_step_operator.py +1 -0
  344. zenml/integrations/spark/step_operators/spark_entrypoint_configuration.py +1 -0
  345. zenml/integrations/spark/step_operators/spark_step_operator.py +2 -0
  346. zenml/integrations/tekton/__init__.py +2 -1
  347. zenml/integrations/tekton/flavors/tekton_orchestrator_flavor.py +66 -23
  348. zenml/integrations/tekton/orchestrators/tekton_orchestrator.py +549 -235
  349. zenml/integrations/tensorboard/__init__.py +1 -13
  350. zenml/integrations/tensorboard/services/tensorboard_service.py +3 -4
  351. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +66 -59
  352. zenml/integrations/tensorflow/__init__.py +10 -15
  353. zenml/integrations/tensorflow/materializers/keras_materializer.py +24 -27
  354. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +9 -16
  355. zenml/integrations/vllm/__init__.py +50 -0
  356. zenml/integrations/vllm/flavors/__init__.py +21 -0
  357. zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
  358. zenml/integrations/vllm/model_deployers/__init__.py +19 -0
  359. zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
  360. zenml/integrations/vllm/services/__init__.py +19 -0
  361. zenml/integrations/vllm/services/vllm_deployment.py +206 -0
  362. zenml/integrations/wandb/__init__.py +1 -0
  363. zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +21 -18
  364. zenml/integrations/whylogs/__init__.py +18 -2
  365. zenml/integrations/whylogs/data_validators/whylogs_data_validator.py +3 -3
  366. zenml/integrations/whylogs/flavors/whylogs_data_validator_flavor.py +1 -1
  367. zenml/integrations/whylogs/materializers/whylogs_materializer.py +14 -21
  368. zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
  369. zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
  370. zenml/io/fileio.py +1 -0
  371. zenml/io/filesystem.py +2 -2
  372. zenml/io/local_filesystem.py +3 -3
  373. zenml/logger.py +41 -17
  374. zenml/logging/__init__.py +5 -0
  375. zenml/logging/step_logging.py +280 -32
  376. zenml/{new/steps → login}/__init__.py +4 -1
  377. zenml/login/credentials.py +370 -0
  378. zenml/login/credentials_store.py +638 -0
  379. zenml/{new/pipelines → login/pro}/__init__.py +4 -1
  380. zenml/login/pro/client.py +492 -0
  381. zenml/login/pro/constants.py +28 -0
  382. zenml/{zen_stores/schemas/identity_schemas.py → login/pro/models.py} +9 -11
  383. zenml/login/pro/organization/__init__.py +14 -0
  384. zenml/login/pro/organization/client.py +79 -0
  385. zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
  386. zenml/{_hub → login/pro/tenant}/__init__.py +2 -2
  387. zenml/login/pro/tenant/client.py +92 -0
  388. zenml/login/pro/tenant/models.py +176 -0
  389. zenml/login/pro/utils.py +107 -0
  390. zenml/login/server_info.py +52 -0
  391. zenml/{cli → login}/web_login.py +71 -21
  392. zenml/materializers/__init__.py +2 -4
  393. zenml/materializers/base_materializer.py +180 -51
  394. zenml/materializers/built_in_materializer.py +50 -23
  395. zenml/materializers/cloudpickle_materializer.py +4 -5
  396. zenml/materializers/numpy_materializer.py +23 -233
  397. zenml/materializers/pandas_materializer.py +22 -174
  398. zenml/materializers/pydantic_materializer.py +2 -2
  399. zenml/materializers/service_materializer.py +12 -10
  400. zenml/materializers/structured_string_materializer.py +12 -6
  401. zenml/materializers/uuid_materializer.py +79 -0
  402. zenml/metadata/lazy_load.py +33 -17
  403. zenml/metadata/metadata_types.py +112 -3
  404. zenml/model/lazy_load.py +85 -4
  405. zenml/model/model.py +236 -226
  406. zenml/model/utils.py +99 -141
  407. zenml/model_deployers/base_model_deployer.py +319 -47
  408. zenml/model_registries/base_model_registry.py +17 -15
  409. zenml/models/__init__.py +254 -175
  410. zenml/models/v2/base/base.py +254 -112
  411. zenml/models/v2/base/base_plugin_flavor.py +76 -0
  412. zenml/models/v2/base/filter.py +530 -199
  413. zenml/models/v2/base/page.py +2 -12
  414. zenml/models/v2/base/scoped.py +292 -22
  415. zenml/models/v2/core/action.py +276 -0
  416. zenml/models/v2/core/action_flavor.py +57 -0
  417. zenml/models/v2/core/api_key.py +35 -12
  418. zenml/models/v2/core/artifact.py +126 -6
  419. zenml/models/v2/core/artifact_version.py +289 -65
  420. zenml/models/v2/core/artifact_visualization.py +10 -4
  421. zenml/models/v2/core/code_reference.py +13 -4
  422. zenml/models/v2/core/code_repository.py +39 -16
  423. zenml/models/v2/core/component.py +113 -61
  424. zenml/models/v2/core/device.py +25 -5
  425. zenml/models/v2/core/event_source.py +244 -0
  426. zenml/models/v2/core/event_source_flavor.py +67 -0
  427. zenml/models/v2/core/flavor.py +90 -27
  428. zenml/models/v2/core/logs.py +81 -12
  429. zenml/models/v2/core/model.py +91 -42
  430. zenml/models/v2/core/model_version.py +100 -55
  431. zenml/models/v2/core/model_version_artifact.py +96 -89
  432. zenml/models/v2/core/model_version_pipeline_run.py +86 -53
  433. zenml/models/v2/core/pipeline.py +205 -80
  434. zenml/models/v2/core/pipeline_build.py +110 -21
  435. zenml/models/v2/core/pipeline_deployment.py +101 -36
  436. zenml/models/v2/core/pipeline_run.py +545 -33
  437. zenml/models/v2/core/run_metadata.py +23 -181
  438. zenml/models/v2/core/run_template.py +439 -0
  439. zenml/models/v2/core/schedule.py +66 -29
  440. zenml/models/v2/core/secret.py +33 -21
  441. zenml/models/v2/core/server_settings.py +224 -0
  442. zenml/models/v2/core/service.py +500 -0
  443. zenml/models/v2/core/service_account.py +40 -15
  444. zenml/models/v2/core/service_connector.py +247 -49
  445. zenml/models/v2/core/stack.py +163 -70
  446. zenml/models/v2/core/step_run.py +210 -48
  447. zenml/models/v2/core/tag.py +21 -8
  448. zenml/models/v2/core/tag_resource.py +13 -4
  449. zenml/models/v2/core/trigger.py +422 -0
  450. zenml/models/v2/core/trigger_execution.py +119 -0
  451. zenml/models/v2/core/user.py +136 -69
  452. zenml/models/v2/core/workspace.py +26 -7
  453. zenml/models/v2/misc/auth_models.py +11 -2
  454. zenml/models/v2/misc/build_item.py +3 -3
  455. zenml/models/v2/misc/external_user.py +3 -6
  456. zenml/models/v2/misc/info_models.py +78 -0
  457. zenml/models/v2/misc/loaded_visualization.py +2 -2
  458. zenml/models/v2/misc/run_metadata.py +38 -0
  459. zenml/models/v2/misc/server_models.py +100 -0
  460. zenml/models/v2/misc/service_connector_type.py +9 -17
  461. zenml/models/v2/misc/stack_deployment.py +96 -0
  462. zenml/models/v2/misc/user_auth.py +7 -9
  463. zenml/orchestrators/__init__.py +4 -0
  464. zenml/orchestrators/base_orchestrator.py +136 -25
  465. zenml/orchestrators/containerized_orchestrator.py +1 -0
  466. zenml/orchestrators/dag_runner.py +18 -3
  467. zenml/orchestrators/input_utils.py +109 -48
  468. zenml/orchestrators/local/local_orchestrator.py +10 -0
  469. zenml/orchestrators/local_docker/local_docker_orchestrator.py +14 -42
  470. zenml/orchestrators/output_utils.py +16 -6
  471. zenml/orchestrators/publish_utils.py +12 -5
  472. zenml/orchestrators/step_launcher.py +142 -194
  473. zenml/orchestrators/step_run_utils.py +386 -0
  474. zenml/orchestrators/step_runner.py +181 -270
  475. zenml/orchestrators/utils.py +219 -84
  476. zenml/orchestrators/wheeled_orchestrator.py +147 -0
  477. zenml/pipelines/__init__.py +3 -16
  478. zenml/{new/pipelines → pipelines}/build_utils.py +287 -47
  479. zenml/{new/pipelines → pipelines}/pipeline_context.py +6 -2
  480. zenml/pipelines/pipeline_decorator.py +40 -64
  481. zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +376 -440
  482. zenml/pipelines/run_utils.py +358 -0
  483. zenml/plugins/__init__.py +0 -0
  484. zenml/plugins/base_plugin_flavor.py +88 -0
  485. zenml/plugins/plugin_flavor_registry.py +342 -0
  486. zenml/secret/base_secret.py +7 -8
  487. zenml/secret/schemas/basic_auth_secret_schema.py +0 -1
  488. zenml/service_connectors/docker_service_connector.py +19 -4
  489. zenml/service_connectors/service_connector.py +12 -14
  490. zenml/service_connectors/service_connector_registry.py +71 -55
  491. zenml/service_connectors/service_connector_utils.py +418 -0
  492. zenml/services/__init__.py +0 -2
  493. zenml/services/container/container_service.py +9 -6
  494. zenml/services/container/container_service_endpoint.py +1 -1
  495. zenml/services/container/entrypoint.py +3 -2
  496. zenml/services/local/local_daemon_entrypoint.py +9 -6
  497. zenml/services/local/local_service.py +1 -1
  498. zenml/services/local/local_service_endpoint.py +1 -1
  499. zenml/services/service.py +222 -130
  500. zenml/services/service_status.py +2 -1
  501. zenml/services/service_type.py +6 -5
  502. zenml/stack/flavor.py +25 -18
  503. zenml/stack/flavor_registry.py +4 -4
  504. zenml/stack/stack.py +20 -131
  505. zenml/stack/stack_component.py +136 -110
  506. zenml/stack/utils.py +36 -15
  507. zenml/stack_deployments/__init__.py +14 -0
  508. zenml/stack_deployments/aws_stack_deployment.py +320 -0
  509. zenml/stack_deployments/azure_stack_deployment.py +315 -0
  510. zenml/stack_deployments/gcp_stack_deployment.py +315 -0
  511. zenml/stack_deployments/stack_deployment.py +232 -0
  512. zenml/stack_deployments/utils.py +48 -0
  513. zenml/step_operators/step_operator_entrypoint_configuration.py +2 -1
  514. zenml/steps/__init__.py +3 -9
  515. zenml/steps/base_step.py +172 -315
  516. zenml/{new/steps → steps}/decorated_step.py +1 -0
  517. zenml/steps/entrypoint_function_utils.py +33 -93
  518. zenml/{new/steps → steps}/step_context.py +70 -50
  519. zenml/steps/step_decorator.py +47 -93
  520. zenml/steps/step_invocation.py +22 -60
  521. zenml/steps/utils.py +161 -48
  522. zenml/types.py +14 -1
  523. zenml/utils/archivable.py +178 -0
  524. zenml/utils/callback_registry.py +71 -0
  525. zenml/utils/code_repository_utils.py +1 -0
  526. zenml/utils/code_utils.py +346 -0
  527. zenml/utils/cuda_utils.py +50 -0
  528. zenml/utils/dashboard_utils.py +67 -21
  529. zenml/utils/deprecation_utils.py +22 -24
  530. zenml/utils/dict_utils.py +22 -0
  531. zenml/utils/docker_utils.py +34 -5
  532. zenml/utils/downloaded_repository_context.py +1 -0
  533. zenml/utils/env_utils.py +55 -1
  534. zenml/utils/filesync_model.py +65 -28
  535. zenml/utils/function_utils.py +260 -0
  536. zenml/utils/integration_utils.py +1 -0
  537. zenml/utils/json_utils.py +131 -0
  538. zenml/utils/materializer_utils.py +1 -1
  539. zenml/utils/metadata_utils.py +368 -0
  540. zenml/utils/notebook_utils.py +136 -0
  541. zenml/utils/package_utils.py +89 -0
  542. zenml/utils/pagination_utils.py +9 -7
  543. zenml/utils/pipeline_docker_image_builder.py +152 -149
  544. zenml/utils/pydantic_utils.py +276 -66
  545. zenml/utils/requirements_utils.py +71 -0
  546. zenml/utils/secret_utils.py +66 -12
  547. zenml/utils/settings_utils.py +2 -1
  548. zenml/utils/singleton.py +15 -3
  549. zenml/utils/source_code_utils.py +1 -0
  550. zenml/utils/source_utils.py +236 -14
  551. zenml/utils/string_utils.py +140 -0
  552. zenml/utils/typed_model.py +5 -3
  553. zenml/utils/typing_utils.py +223 -0
  554. zenml/utils/visualization_utils.py +5 -3
  555. zenml/utils/yaml_utils.py +1 -1
  556. zenml/zen_server/auth.py +387 -55
  557. zenml/zen_server/cache.py +208 -0
  558. zenml/zen_server/cloud_utils.py +253 -0
  559. zenml/zen_server/csrf.py +91 -0
  560. zenml/zen_server/dashboard/assets/404-Dfq64Boz.js +1 -0
  561. zenml/zen_server/dashboard/assets/@radix-DeK6qiuw.js +85 -0
  562. zenml/zen_server/dashboard/assets/@react-router-B3Z5rLr2.js +29 -0
  563. zenml/zen_server/dashboard/assets/@reactflow-BUNIMFeC.js +17 -0
  564. zenml/zen_server/dashboard/assets/@reactflow-C26Olbza.css +1 -0
  565. zenml/zen_server/dashboard/assets/@tanstack-DT5WLu9C.js +22 -0
  566. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-B73Vs10T.js +1 -0
  567. zenml/zen_server/dashboard/assets/CodeSnippet-Bbx6fIb6.css +1 -0
  568. zenml/zen_server/dashboard/assets/CodeSnippet-DIJRT2NT.js +9 -0
  569. zenml/zen_server/dashboard/assets/CollapsibleCard-BzUHGZOU.js +1 -0
  570. zenml/zen_server/dashboard/assets/Commands-BEGyld4c.js +1 -0
  571. zenml/zen_server/dashboard/assets/ComponentBadge-xyKiek1s.js +1 -0
  572. zenml/zen_server/dashboard/assets/CopyButton-DhW-mapu.js +2 -0
  573. zenml/zen_server/dashboard/assets/CsvVizualization-D8oazBiE.js +15 -0
  574. zenml/zen_server/dashboard/assets/DeleteAlertDialog-WkSIIgfy.js +1 -0
  575. zenml/zen_server/dashboard/assets/DialogItem-Bgroeg29.js +1 -0
  576. zenml/zen_server/dashboard/assets/DisplayDate-CDMUcQHS.js +1 -0
  577. zenml/zen_server/dashboard/assets/EmptyState-BzdlCwp3.js +1 -0
  578. zenml/zen_server/dashboard/assets/Error-CY5tlu17.js +1 -0
  579. zenml/zen_server/dashboard/assets/ExecutionStatus-G8mjIaeA.js +1 -0
  580. zenml/zen_server/dashboard/assets/Helpbox-Bb1ed--O.js +1 -0
  581. zenml/zen_server/dashboard/assets/Infobox-Da6-76M2.js +1 -0
  582. zenml/zen_server/dashboard/assets/InlineAvatar-DqnZaBNq.js +1 -0
  583. zenml/zen_server/dashboard/assets/Lock-CYYy18Mm.js +1 -0
  584. zenml/zen_server/dashboard/assets/MarkdownVisualization-ylXaAxev.js +14 -0
  585. zenml/zen_server/dashboard/assets/NestedCollapsible-aK5ojKoF.js +1 -0
  586. zenml/zen_server/dashboard/assets/NumberBox-Dtp3J6g5.js +1 -0
  587. zenml/zen_server/dashboard/assets/Partials-CqZp5NMX.js +1 -0
  588. zenml/zen_server/dashboard/assets/PasswordChecker-B0nadgh6.js +1 -0
  589. zenml/zen_server/dashboard/assets/ProBadge-B4tRUYve.js +1 -0
  590. zenml/zen_server/dashboard/assets/ProCta-CZuP29Qz.js +1 -0
  591. zenml/zen_server/dashboard/assets/ProviderIcon-Bd7GUQ1_.js +1 -0
  592. zenml/zen_server/dashboard/assets/ProviderRadio-mstdqzsS.js +1 -0
  593. zenml/zen_server/dashboard/assets/RunSelector-CsruSB4i.js +1 -0
  594. zenml/zen_server/dashboard/assets/RunsBody-DxxtWVYz.js +1 -0
  595. zenml/zen_server/dashboard/assets/SearchField-D6tPxyqw.js +1 -0
  596. zenml/zen_server/dashboard/assets/SecretTooltip-CLzJIYW_.js +1 -0
  597. zenml/zen_server/dashboard/assets/SetPassword-Yn50ooBC.js +1 -0
  598. zenml/zen_server/dashboard/assets/StackList-U537qoYd.js +1 -0
  599. zenml/zen_server/dashboard/assets/Tabs-CNv-eTYM.js +1 -0
  600. zenml/zen_server/dashboard/assets/Tick-jEIevzVf.js +1 -0
  601. zenml/zen_server/dashboard/assets/UpdatePasswordSchemas-C16GW-kX.js +1 -0
  602. zenml/zen_server/dashboard/assets/UsageReason-Bf2tzhv1.js +1 -0
  603. zenml/zen_server/dashboard/assets/WizardFooter-D6i-AP1K.js +1 -0
  604. zenml/zen_server/dashboard/assets/acp-DOsXjFc7.webp +0 -0
  605. zenml/zen_server/dashboard/assets/adam-e-y0WnB_.webp +0 -0
  606. zenml/zen_server/dashboard/assets/alex-DcCuDHPg.webp +0 -0
  607. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-DUti43aF.js +1 -0
  608. zenml/zen_server/dashboard/assets/baris-C0ZrZ10g.webp +0 -0
  609. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  610. zenml/zen_server/dashboard/assets/check-circle-jNbX5-sR.js +1 -0
  611. zenml/zen_server/dashboard/assets/chevron-down-6JyMkfjR.js +1 -0
  612. zenml/zen_server/dashboard/assets/chevron-right-double-D7ojK9Co.js +1 -0
  613. zenml/zen_server/dashboard/assets/cloud-squares-DeRLMopf.svg +43 -0
  614. zenml/zen_server/dashboard/assets/code-browser-CUFUIHfp.js +1 -0
  615. zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
  616. zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
  617. zenml/zen_server/dashboard/assets/connectors-video-C9qY4syJ.svg +21 -0
  618. zenml/zen_server/dashboard/assets/copy-C8XQA2Ug.js +1 -0
  619. zenml/zen_server/dashboard/assets/create-stack-Ch2WPs9U.js +1 -0
  620. zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
  621. zenml/zen_server/dashboard/assets/delete-run-Byf9hTjA.js +1 -0
  622. zenml/zen_server/dashboard/assets/docker-BdA9vrnW.js +1 -0
  623. zenml/zen_server/dashboard/assets/dots-horizontal-otGBOSDJ.js +1 -0
  624. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  625. zenml/zen_server/dashboard/assets/form-schemas-BZqKBPBF.js +1 -0
  626. zenml/zen_server/dashboard/assets/gcp-CFtm4BA7.js +1 -0
  627. zenml/zen_server/dashboard/assets/hamza-NKKOZz1I.webp +0 -0
  628. zenml/zen_server/dashboard/assets/help-Cc9bBIJH.js +1 -0
  629. zenml/zen_server/dashboard/assets/index-CE0aQlv8.js +55 -0
  630. zenml/zen_server/dashboard/assets/index-CtdYkjUi.js +1 -0
  631. zenml/zen_server/dashboard/assets/index-CyBKZcpO.js +1 -0
  632. zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
  633. zenml/zen_server/dashboard/assets/index-Uu49AX48.js +1 -0
  634. zenml/zen_server/dashboard/assets/index-v6gQjDEo.js +1 -0
  635. zenml/zen_server/dashboard/assets/index.esm-Dy6Z9Ung.js +1 -0
  636. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
  637. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
  638. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
  639. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
  640. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
  641. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
  642. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
  643. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
  644. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
  645. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
  646. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
  647. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
  648. zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
  649. zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
  650. zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
  651. zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
  652. zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
  653. zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
  654. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
  655. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
  656. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
  657. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
  658. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
  659. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
  660. zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
  661. zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
  662. zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
  663. zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
  664. zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
  665. zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
  666. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
  667. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
  668. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
  669. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
  670. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
  671. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
  672. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
  673. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
  674. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
  675. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
  676. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
  677. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
  678. zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
  679. zenml/zen_server/dashboard/assets/kubernetes-B2wmAJ1d.js +1 -0
  680. zenml/zen_server/dashboard/assets/layout-BtHBmE4w.js +1 -0
  681. zenml/zen_server/dashboard/assets/link-external-b9AXw_sW.js +1 -0
  682. zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
  683. zenml/zen_server/dashboard/assets/login-mutation-DNDVp_2H.js +1 -0
  684. zenml/zen_server/dashboard/assets/logs-WMSM52RF.js +1 -0
  685. zenml/zen_server/dashboard/assets/mcp-Cb1aMeoq.webp +0 -0
  686. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  687. zenml/zen_server/dashboard/assets/not-found-Bmup4ctE.js +1 -0
  688. zenml/zen_server/dashboard/assets/package-C6uypY4h.js +1 -0
  689. zenml/zen_server/dashboard/assets/page--XLMzHrn.js +1 -0
  690. zenml/zen_server/dashboard/assets/page-ANYGfEUL.js +1 -0
  691. zenml/zen_server/dashboard/assets/page-B5Sr8pib.js +1 -0
  692. zenml/zen_server/dashboard/assets/page-BC27C_OI.js +2 -0
  693. zenml/zen_server/dashboard/assets/page-BNxYrN0q.js +1 -0
  694. zenml/zen_server/dashboard/assets/page-BYJfqgLN.js +1 -0
  695. zenml/zen_server/dashboard/assets/page-B_0XkV48.js +1 -0
  696. zenml/zen_server/dashboard/assets/page-BrmJp1Wt.js +1 -0
  697. zenml/zen_server/dashboard/assets/page-C2nU3Gxn.js +1 -0
  698. zenml/zen_server/dashboard/assets/page-C70wZtV2.js +1 -0
  699. zenml/zen_server/dashboard/assets/page-CHRn1fQm.js +1 -0
  700. zenml/zen_server/dashboard/assets/page-CWr96ZKN.js +1 -0
  701. zenml/zen_server/dashboard/assets/page-CXAbSyp9.js +1 -0
  702. zenml/zen_server/dashboard/assets/page-CaeI9ptC.js +1 -0
  703. zenml/zen_server/dashboard/assets/page-Cc8ZEuj4.js +1 -0
  704. zenml/zen_server/dashboard/assets/page-CltCNL0T.js +1 -0
  705. zenml/zen_server/dashboard/assets/page-CmlYj7Nl.js +1 -0
  706. zenml/zen_server/dashboard/assets/page-D6Ev5P8V.js +1 -0
  707. zenml/zen_server/dashboard/assets/page-D9Oh05fl.js +1 -0
  708. zenml/zen_server/dashboard/assets/page-DGlm1RVc.js +1 -0
  709. zenml/zen_server/dashboard/assets/page-DN4BVIOL.js +1 -0
  710. zenml/zen_server/dashboard/assets/page-Dif8CWyZ.js +1 -0
  711. zenml/zen_server/dashboard/assets/page-DlIi5ThM.js +1 -0
  712. zenml/zen_server/dashboard/assets/page-DoW7YxTu.js +1 -0
  713. zenml/zen_server/dashboard/assets/page-Dth9X1Ih.js +1 -0
  714. zenml/zen_server/dashboard/assets/page-DweqqCkF.js +1 -0
  715. zenml/zen_server/dashboard/assets/page-DyOJ_pq3.js +1 -0
  716. zenml/zen_server/dashboard/assets/page-Hn8q9iJZ.js +1 -0
  717. zenml/zen_server/dashboard/assets/page-IhckKFnD.js +6 -0
  718. zenml/zen_server/dashboard/assets/page-LyZ_l8vR.js +1 -0
  719. zenml/zen_server/dashboard/assets/page-PamGpk0j.js +1 -0
  720. zenml/zen_server/dashboard/assets/page-PxOWfKgF.js +2 -0
  721. zenml/zen_server/dashboard/assets/persist-DeXRG61d.js +1 -0
  722. zenml/zen_server/dashboard/assets/persist-vP0-Xl4f.js +1 -0
  723. zenml/zen_server/dashboard/assets/plus-tf1V2hTJ.js +1 -0
  724. zenml/zen_server/dashboard/assets/refresh-BjOeWlEq.js +1 -0
  725. zenml/zen_server/dashboard/assets/repos-video-D8kpu60k.svg +9 -0
  726. zenml/zen_server/dashboard/assets/rocket-DjT2cDvG.js +1 -0
  727. zenml/zen_server/dashboard/assets/service-DH_oUqQj.js +2 -0
  728. zenml/zen_server/dashboard/assets/settings_preview-0JLrRgHP.webp +0 -0
  729. zenml/zen_server/dashboard/assets/sharedSchema-Bw1_Wa7l.js +14 -0
  730. zenml/zen_server/dashboard/assets/stack-detail-query-B_0R_fd6.js +1 -0
  731. zenml/zen_server/dashboard/assets/stefan-B08Ftbba.webp +0 -0
  732. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  733. zenml/zen_server/dashboard/assets/tick-circle-BEX_Tp4v.js +1 -0
  734. zenml/zen_server/dashboard/assets/tour-cover-BYfeen6M.webp +0 -0
  735. zenml/zen_server/dashboard/assets/trash-arLUMWMS.js +1 -0
  736. zenml/zen_server/dashboard/assets/update-server-settings-mutation-D9qYhfaN.js +1 -0
  737. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  738. zenml/zen_server/dashboard/assets/url-Dh93fvh0.js +1 -0
  739. zenml/zen_server/dashboard/assets/zod-BwEbpOxH.js +1 -0
  740. zenml/zen_server/dashboard/index.html +19 -1
  741. zenml/zen_server/deploy/__init__.py +7 -16
  742. zenml/zen_server/deploy/base_provider.py +49 -78
  743. zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
  744. zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +48 -66
  745. zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +78 -62
  746. zenml/zen_server/deploy/deployer.py +94 -175
  747. zenml/zen_server/deploy/deployment.py +23 -17
  748. zenml/zen_server/deploy/docker/docker_provider.py +15 -31
  749. zenml/zen_server/deploy/docker/docker_zen_server.py +30 -35
  750. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  751. zenml/zen_server/deploy/helm/README.md +3 -13
  752. zenml/zen_server/deploy/helm/templates/NOTES.txt +23 -7
  753. zenml/zen_server/deploy/helm/templates/_environment.tpl +175 -23
  754. zenml/zen_server/deploy/helm/templates/server-db-job.yaml +45 -18
  755. zenml/zen_server/deploy/helm/templates/server-db-pvc.yaml +25 -0
  756. zenml/zen_server/deploy/helm/templates/server-deployment.yaml +22 -6
  757. zenml/zen_server/deploy/helm/templates/server-secret.yaml +11 -10
  758. zenml/zen_server/deploy/helm/values.yaml +210 -28
  759. zenml/zen_server/exceptions.py +20 -1
  760. zenml/zen_server/feature_gate/__init__.py +13 -0
  761. zenml/zen_server/feature_gate/endpoint_utils.py +61 -0
  762. zenml/zen_server/feature_gate/feature_gate_interface.py +49 -0
  763. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +125 -0
  764. zenml/zen_server/jwt.py +64 -32
  765. zenml/zen_server/rate_limit.py +200 -0
  766. zenml/zen_server/rbac/endpoint_utils.py +92 -9
  767. zenml/zen_server/rbac/models.py +21 -17
  768. zenml/zen_server/rbac/rbac_sql_zen_store.py +175 -0
  769. zenml/zen_server/rbac/utils.py +71 -30
  770. zenml/zen_server/rbac/zenml_cloud_rbac.py +13 -188
  771. zenml/zen_server/routers/actions_endpoints.py +324 -0
  772. zenml/zen_server/routers/artifact_version_endpoints.py +28 -2
  773. zenml/zen_server/routers/auth_endpoints.py +249 -131
  774. zenml/zen_server/routers/code_repositories_endpoints.py +1 -0
  775. zenml/zen_server/routers/devices_endpoints.py +56 -49
  776. zenml/zen_server/routers/event_source_endpoints.py +327 -0
  777. zenml/zen_server/routers/logs_endpoints.py +66 -0
  778. zenml/zen_server/routers/model_versions_endpoints.py +59 -0
  779. zenml/zen_server/routers/models_endpoints.py +7 -1
  780. zenml/zen_server/routers/pipeline_builds_endpoints.py +6 -1
  781. zenml/zen_server/routers/pipeline_deployments_endpoints.py +35 -0
  782. zenml/zen_server/routers/pipelines_endpoints.py +19 -32
  783. zenml/zen_server/routers/plugin_endpoints.py +107 -0
  784. zenml/zen_server/routers/run_templates_endpoints.py +212 -0
  785. zenml/zen_server/routers/runs_endpoints.py +91 -32
  786. zenml/zen_server/routers/schedule_endpoints.py +1 -0
  787. zenml/zen_server/routers/secrets_endpoints.py +4 -2
  788. zenml/zen_server/routers/server_endpoints.py +186 -4
  789. zenml/zen_server/routers/service_connectors_endpoints.py +56 -0
  790. zenml/zen_server/routers/service_endpoints.py +180 -0
  791. zenml/zen_server/routers/stack_components_endpoints.py +2 -1
  792. zenml/zen_server/routers/stack_deployment_endpoints.py +164 -0
  793. zenml/zen_server/routers/steps_endpoints.py +21 -12
  794. zenml/zen_server/routers/triggers_endpoints.py +336 -0
  795. zenml/zen_server/routers/users_endpoints.py +280 -45
  796. zenml/zen_server/routers/webhook_endpoints.py +127 -0
  797. zenml/zen_server/routers/workspaces_endpoints.py +220 -185
  798. zenml/zen_server/secure_headers.py +120 -0
  799. zenml/{new → zen_server/template_execution}/__init__.py +1 -1
  800. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +42 -0
  801. zenml/zen_server/template_execution/utils.py +474 -0
  802. zenml/zen_server/template_execution/workload_manager_interface.py +92 -0
  803. zenml/zen_server/utils.py +374 -74
  804. zenml/zen_server/zen_server_api.py +299 -52
  805. zenml/zen_stores/base_zen_store.py +90 -58
  806. zenml/zen_stores/migrations/alembic.py +22 -9
  807. zenml/zen_stores/migrations/env.py +2 -2
  808. zenml/zen_stores/migrations/utils.py +731 -0
  809. zenml/zen_stores/migrations/versions/0.21.0_release.py +0 -1
  810. zenml/zen_stores/migrations/versions/0.21.1_release.py +0 -1
  811. zenml/zen_stores/migrations/versions/0.22.0_release.py +0 -1
  812. zenml/zen_stores/migrations/versions/0.23.0_release.py +0 -1
  813. zenml/zen_stores/migrations/versions/0.30.0_release.py +0 -1
  814. zenml/zen_stores/migrations/versions/0.31.0_release.py +0 -1
  815. zenml/zen_stores/migrations/versions/0.31.1_release.py +0 -1
  816. zenml/zen_stores/migrations/versions/0.32.0_release.py +0 -1
  817. zenml/zen_stores/migrations/versions/0.32.1_release.py +0 -1
  818. zenml/zen_stores/migrations/versions/0.33.0_release.py +0 -1
  819. zenml/zen_stores/migrations/versions/0.34.0_release.py +0 -1
  820. zenml/zen_stores/migrations/versions/0.35.0_release.py +0 -1
  821. zenml/zen_stores/migrations/versions/0.35.1_release.py +0 -1
  822. zenml/zen_stores/migrations/versions/0.36.0_release.py +0 -1
  823. zenml/zen_stores/migrations/versions/0.36.1_release.py +0 -1
  824. zenml/zen_stores/migrations/versions/0.37.0_release.py +0 -1
  825. zenml/zen_stores/migrations/versions/0.38.0_release.py +0 -1
  826. zenml/zen_stores/migrations/versions/0.39.0_release.py +0 -1
  827. zenml/zen_stores/migrations/versions/0.39.1_release.py +0 -1
  828. zenml/zen_stores/migrations/versions/0.40.0_release.py +0 -1
  829. zenml/zen_stores/migrations/versions/0.40.1_release.py +0 -1
  830. zenml/zen_stores/migrations/versions/0.40.2_release.py +0 -1
  831. zenml/zen_stores/migrations/versions/0.40.3_release.py +0 -1
  832. zenml/zen_stores/migrations/versions/0.41.0_release.py +0 -1
  833. zenml/zen_stores/migrations/versions/0.42.0_release.py +0 -1
  834. zenml/zen_stores/migrations/versions/0.42.1_release.py +0 -1
  835. zenml/zen_stores/migrations/versions/0.43.0_release.py +0 -1
  836. zenml/zen_stores/migrations/versions/0.44.0_release.py +0 -1
  837. zenml/zen_stores/migrations/versions/0.44.1_release.py +0 -1
  838. zenml/zen_stores/migrations/versions/0.44.2_release.py +0 -1
  839. zenml/zen_stores/migrations/versions/0.44.3_release.py +0 -1
  840. zenml/zen_stores/migrations/versions/0.45.0_release.py +0 -1
  841. zenml/zen_stores/migrations/versions/0.45.1_release_0_45_1.py +0 -1
  842. zenml/zen_stores/migrations/versions/0.45.2_release.py +0 -1
  843. zenml/zen_stores/migrations/versions/0.45.3_release.py +0 -1
  844. zenml/zen_stores/migrations/versions/0.45.4_release.py +0 -1
  845. zenml/zen_stores/migrations/versions/0.45.5_release.py +0 -1
  846. zenml/zen_stores/migrations/versions/0.45.6_release.py +0 -1
  847. zenml/zen_stores/migrations/versions/0.46.0_release.py +0 -1
  848. zenml/zen_stores/migrations/versions/0.46.1_release.py +0 -1
  849. zenml/zen_stores/migrations/versions/0.47.0_release.py +0 -1
  850. zenml/zen_stores/migrations/versions/0.50.0_release.py +0 -1
  851. zenml/zen_stores/migrations/versions/0.51.0_release.py +0 -1
  852. zenml/zen_stores/migrations/versions/0.52.0_release.py +0 -1
  853. zenml/zen_stores/migrations/versions/0.53.0_release.py +0 -1
  854. zenml/zen_stores/migrations/versions/0.53.1_release.py +0 -1
  855. zenml/zen_stores/migrations/versions/0.54.0_release.py +0 -1
  856. zenml/zen_stores/migrations/versions/0.54.1_release.py +0 -1
  857. zenml/zen_stores/migrations/versions/0.55.0_release.py +0 -1
  858. zenml/zen_stores/migrations/versions/0.55.1_release.py +23 -0
  859. zenml/zen_stores/migrations/versions/0.55.2_release.py +23 -0
  860. zenml/zen_stores/migrations/versions/0.55.3_release.py +23 -0
  861. zenml/zen_stores/migrations/versions/0.55.4_release.py +23 -0
  862. zenml/zen_stores/migrations/versions/0.55.5_release.py +23 -0
  863. zenml/zen_stores/migrations/versions/0.56.0_release.py +23 -0
  864. zenml/zen_stores/migrations/versions/0.56.1_release.py +23 -0
  865. zenml/zen_stores/migrations/versions/0.56.2_release.py +23 -0
  866. zenml/zen_stores/migrations/versions/0.56.3_release.py +23 -0
  867. zenml/zen_stores/migrations/versions/0.56.4_release.py +23 -0
  868. zenml/zen_stores/migrations/versions/0.57.0.rc1_release.py +23 -0
  869. zenml/zen_stores/migrations/versions/0.57.0.rc2_release.py +23 -0
  870. zenml/zen_stores/migrations/versions/0.57.0_release.py +23 -0
  871. zenml/zen_stores/migrations/versions/0.57.1_release.py +23 -0
  872. zenml/zen_stores/migrations/versions/0.58.0_release.py +23 -0
  873. zenml/zen_stores/migrations/versions/0.58.1_release.py +23 -0
  874. zenml/zen_stores/migrations/versions/0.58.2_release.py +23 -0
  875. zenml/zen_stores/migrations/versions/0.60.0_release.py +23 -0
  876. zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
  877. zenml/zen_stores/migrations/versions/0.62.0_release.py +23 -0
  878. zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
  879. zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
  880. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  881. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  882. zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
  883. zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
  884. zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
  885. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  886. zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
  887. zenml/zen_stores/migrations/versions/0.72.0_release.py +23 -0
  888. zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
  889. zenml/zen_stores/migrations/versions/03742aa7fdd7_add_secrets.py +1 -0
  890. zenml/zen_stores/migrations/versions/0701da9951a0_added_service_table.py +94 -0
  891. zenml/zen_stores/migrations/versions/0b06faa59c93_add_service_connectors.py +1 -0
  892. zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
  893. zenml/zen_stores/migrations/versions/0e4735b23577_increase_pipeline_spec_field_length.py +1 -0
  894. zenml/zen_stores/migrations/versions/1041bc644e0d_remove_secrets_manager.py +6 -3
  895. zenml/zen_stores/migrations/versions/10a907dad202_delete_mlmd_tables.py +2 -1
  896. zenml/zen_stores/migrations/versions/14d687c8fa1c_rename_model_config_to_model_version.py +1 -0
  897. zenml/zen_stores/migrations/versions/19f27d5b234e_add_build_and_deployment_tables.py +1 -0
  898. zenml/zen_stores/migrations/versions/1a9a9d2a836d_admin_users.py +56 -0
  899. zenml/zen_stores/migrations/versions/1ac1b9c04da1_make_secrets_values_optional.py +1 -0
  900. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  901. zenml/zen_stores/migrations/versions/1d74e596abb8_add_run_once_start_time_to_schedule.py +36 -0
  902. zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
  903. zenml/zen_stores/migrations/versions/248dfd320b68_update_size_of_flavor_config_schema.py +1 -0
  904. zenml/zen_stores/migrations/versions/25155145c545_separate_actions_and_triggers.py +228 -0
  905. zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
  906. zenml/zen_stores/migrations/versions/26b776ad583e_redesign_artifacts.py +9 -10
  907. zenml/zen_stores/migrations/versions/2d201872e23c_remove_db_dependency_loop.py +29 -0
  908. zenml/zen_stores/migrations/versions/37835ce041d2_optimizing_database.py +4 -3
  909. zenml/zen_stores/migrations/versions/389046140cad_data_versioning.py +1 -0
  910. zenml/zen_stores/migrations/versions/3944116bbd56_rename_project_to_workspace.py +1 -0
  911. zenml/zen_stores/migrations/versions/3b68abe58f44_add_model_watchtower_entities.py +1 -0
  912. zenml/zen_stores/migrations/versions/3c5a367730c2_add_environment_info_to_runs.py +1 -0
  913. zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
  914. zenml/zen_stores/migrations/versions/43a86093b60e_add_labels_for_stack_components.py +1 -0
  915. zenml/zen_stores/migrations/versions/46506f72f0ed_add_server_settings.py +123 -0
  916. zenml/zen_stores/migrations/versions/479103df60b6_add_triggers.py +162 -0
  917. zenml/zen_stores/migrations/versions/4a3087070f4e_add_step_source_code.py +1 -0
  918. zenml/zen_stores/migrations/versions/4c41c0ca42db_add_code_repository_table.py +1 -0
  919. zenml/zen_stores/migrations/versions/4d688d8f7aff_rename_model_version_to_model.py +1 -0
  920. zenml/zen_stores/migrations/versions/4e1972485075_endpoint_artifact_deployment_artifact.py +1 -0
  921. zenml/zen_stores/migrations/versions/4f66af55fbb9_rename_model_config_model_to_model_.py +1 -0
  922. zenml/zen_stores/migrations/versions/5330ba58bf20_rename_tables_and_foreign_keys.py +8 -9
  923. zenml/zen_stores/migrations/versions/5994f9ad0489_introduce_role_permissions.py +4 -2
  924. zenml/zen_stores/migrations/versions/5cc3f41cf048_add_save_models_to_registry.py +1 -0
  925. zenml/zen_stores/migrations/versions/6119cd9b93c2_tags_table.py +1 -0
  926. zenml/zen_stores/migrations/versions/623a234c11f5_add_sdk_docs_url_to_flavors.py +1 -0
  927. zenml/zen_stores/migrations/versions/6917bce75069_add_pipeline_run_unique_constraint.py +5 -4
  928. zenml/zen_stores/migrations/versions/6a28c4fd0ef2_add_caching_info.py +1 -0
  929. zenml/zen_stores/migrations/versions/6f707b385dc1_fix_model_artifacts.py +1 -0
  930. zenml/zen_stores/migrations/versions/722392c91006_make_is_service_account_mandatory.py +1 -0
  931. zenml/zen_stores/migrations/versions/72675226b2de_unique_users.py +31 -0
  932. zenml/zen_stores/migrations/versions/72722dee4686_track_server_version.py +1 -0
  933. zenml/zen_stores/migrations/versions/7280c14811d6_use_text_type.py +1 -0
  934. zenml/zen_stores/migrations/versions/728c6369cfaa_add_name_column_to_input_artifact_pk.py +4 -2
  935. zenml/zen_stores/migrations/versions/729263e47b55_fix_external_input_artifacts.py +1 -0
  936. zenml/zen_stores/migrations/versions/743ec82b1b3c_update_size_of_build_images.py +3 -2
  937. zenml/zen_stores/migrations/versions/7500f434b71c_remove_shared_columns.py +4 -2
  938. zenml/zen_stores/migrations/versions/76a7b9451ccd_add_build_template_deployment_id.py +52 -0
  939. zenml/zen_stores/migrations/versions/7834208cc3f6_artifact_project_scoping.py +9 -7
  940. zenml/zen_stores/migrations/versions/7b651bf6822e_track_secrets_in_db.py +7 -7
  941. zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
  942. zenml/zen_stores/migrations/versions/7e4a481d17f7_add_identity_table.py +3 -2
  943. zenml/zen_stores/migrations/versions/7f603e583dd7_fixed_migration.py +2 -1
  944. zenml/zen_stores/migrations/versions/86fa52918b54_remove_teams_and_roles.py +1 -0
  945. zenml/zen_stores/migrations/versions/8a64fbfecda0_add_num_outputs_to_run_step.py +1 -0
  946. zenml/zen_stores/migrations/versions/8ed03137cacc_polymorthic_run_metadata.py +1 -0
  947. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  948. zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
  949. zenml/zen_stores/migrations/versions/93cbda80a732_add_service_accounts.py +1 -0
  950. zenml/zen_stores/migrations/versions/979eff8fc4b1_add_code_repo_description_and_logo_url.py +1 -0
  951. zenml/zen_stores/migrations/versions/9971237fa937_artifact_visualizations.py +1 -0
  952. zenml/zen_stores/migrations/versions/9d8020441014_increase_step_configuration_length.py +1 -0
  953. zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
  954. zenml/zen_stores/migrations/versions/a39c4184c8ce_remove_secrets_manager_flavors.py +3 -2
  955. zenml/zen_stores/migrations/versions/a91762e6be36_artifact_version_table.py +5 -4
  956. zenml/zen_stores/migrations/versions/ade72effebaf_added_logs_table.py +1 -0
  957. zenml/zen_stores/migrations/versions/alembic_start.py +2 -1
  958. zenml/zen_stores/migrations/versions/b4eccf34dfa3_add_hub_token_to_user_model.py +1 -0
  959. zenml/zen_stores/migrations/versions/b4fca5241eea_migrate_onboarding_state.py +167 -0
  960. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  961. zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
  962. zenml/zen_stores/migrations/versions/b73bc71f1106_remove_component_spec_path.py +36 -0
  963. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  964. zenml/zen_stores/migrations/versions/c1b18cec3a48_increase_length_on_flavor_config_schema.py +1 -0
  965. zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
  966. zenml/zen_stores/migrations/versions/cc269488e5a9_separate_run_metadata.py +135 -0
  967. zenml/zen_stores/migrations/versions/cc9894cb58aa_add_user_metadata.py +41 -0
  968. zenml/zen_stores/migrations/versions/ccd68b7825ae_add_status_to_pipeline_and_step_run.py +1 -0
  969. zenml/zen_stores/migrations/versions/d02b3d3464cf_add_orchestrator_run_id_column.py +1 -0
  970. zenml/zen_stores/migrations/versions/d26471b6fe8f_update_build_filtering.py +1 -0
  971. zenml/zen_stores/migrations/versions/d7b3acf9aa46_create_schedule_table.py +1 -0
  972. zenml/zen_stores/migrations/versions/e1d66d91a099_add_stack_and_component_spec_paths_to_.py +1 -0
  973. zenml/zen_stores/migrations/versions/e5225281b4d3_add_connector_skew_tolerance.py +1 -0
  974. zenml/zen_stores/migrations/versions/e65aa6708ff7_pipeline_versioning.py +1 -0
  975. zenml/zen_stores/migrations/versions/ec0d785ca296_create_run_metadata_table.py +1 -0
  976. zenml/zen_stores/migrations/versions/ec6307720f92_simplify_model_version_links.py +119 -0
  977. zenml/zen_stores/migrations/versions/f3b3964e3a0f_add_oauth_devices.py +1 -0
  978. zenml/zen_stores/migrations/versions/f49904a80aa7_increase_length_of_artifact_table_sources.py +1 -0
  979. zenml/zen_stores/migrations/versions/fbd7f18ced1e_increase_step_run_field_lengths.py +5 -4
  980. zenml/zen_stores/rest_zen_store.py +1326 -305
  981. zenml/zen_stores/schemas/__init__.py +22 -3
  982. zenml/zen_stores/schemas/action_schemas.py +192 -0
  983. zenml/zen_stores/schemas/api_key_schemas.py +23 -10
  984. zenml/zen_stores/schemas/artifact_schemas.py +112 -49
  985. zenml/zen_stores/schemas/artifact_visualization_schemas.py +17 -8
  986. zenml/zen_stores/schemas/base_schemas.py +27 -0
  987. zenml/zen_stores/schemas/code_repository_schemas.py +25 -10
  988. zenml/zen_stores/schemas/component_schemas.py +74 -11
  989. zenml/zen_stores/schemas/constants.py +16 -0
  990. zenml/zen_stores/schemas/device_schemas.py +29 -15
  991. zenml/zen_stores/schemas/event_source_schemas.py +188 -0
  992. zenml/zen_stores/schemas/flavor_schemas.py +19 -9
  993. zenml/zen_stores/schemas/logs_schemas.py +12 -6
  994. zenml/zen_stores/schemas/model_schemas.py +192 -139
  995. zenml/zen_stores/schemas/pipeline_build_schemas.py +16 -16
  996. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +58 -17
  997. zenml/zen_stores/schemas/pipeline_run_schemas.py +170 -35
  998. zenml/zen_stores/schemas/pipeline_schemas.py +51 -33
  999. zenml/zen_stores/schemas/run_metadata_schemas.py +28 -78
  1000. zenml/zen_stores/schemas/run_template_schemas.py +267 -0
  1001. zenml/zen_stores/schemas/schedule_schema.py +15 -5
  1002. zenml/zen_stores/schemas/secret_schemas.py +18 -10
  1003. zenml/zen_stores/schemas/server_settings_schemas.py +129 -0
  1004. zenml/zen_stores/schemas/service_connector_schemas.py +13 -6
  1005. zenml/zen_stores/schemas/service_schemas.py +258 -0
  1006. zenml/zen_stores/schemas/stack_schemas.py +23 -6
  1007. zenml/zen_stores/schemas/step_run_schemas.py +132 -41
  1008. zenml/zen_stores/schemas/tag_schemas.py +31 -50
  1009. zenml/zen_stores/schemas/trigger_schemas.py +316 -0
  1010. zenml/zen_stores/schemas/user_schemas.py +66 -23
  1011. zenml/zen_stores/schemas/utils.py +112 -0
  1012. zenml/zen_stores/schemas/workspace_schemas.py +36 -19
  1013. zenml/zen_stores/secrets_stores/aws_secrets_store.py +41 -32
  1014. zenml/zen_stores/secrets_stores/azure_secrets_store.py +20 -23
  1015. zenml/zen_stores/secrets_stores/base_secrets_store.py +80 -12
  1016. zenml/zen_stores/secrets_stores/gcp_secrets_store.py +42 -33
  1017. zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py +7 -11
  1018. zenml/zen_stores/secrets_stores/secrets_store_interface.py +1 -0
  1019. zenml/zen_stores/secrets_stores/service_connector_secrets_store.py +15 -8
  1020. zenml/zen_stores/secrets_stores/sql_secrets_store.py +8 -9
  1021. zenml/zen_stores/sql_zen_store.py +4062 -799
  1022. zenml/zen_stores/template_utils.py +263 -0
  1023. zenml/zen_stores/zen_store_interface.py +614 -44
  1024. zenml_nightly-0.72.0.dev20250116.dist-info/METADATA +486 -0
  1025. zenml_nightly-0.72.0.dev20250116.dist-info/RECORD +1294 -0
  1026. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/WHEEL +1 -1
  1027. CLA.md +0 -110
  1028. CODE-OF-CONDUCT.md +0 -132
  1029. CONTRIBUTING.md +0 -260
  1030. README.md +0 -304
  1031. RELEASE_NOTES.md +0 -3919
  1032. ROADMAP.md +0 -5
  1033. SECURITY.md +0 -15
  1034. zenml/_hub/client.py +0 -285
  1035. zenml/_hub/constants.py +0 -21
  1036. zenml/_hub/utils.py +0 -80
  1037. zenml/api.py +0 -61
  1038. zenml/cli/hub.py +0 -1115
  1039. zenml/cli/stack_recipes.py +0 -469
  1040. zenml/integrations/gcp/google_cloud_function.py +0 -187
  1041. zenml/integrations/gcp/google_cloud_scheduler.py +0 -83
  1042. zenml/integrations/gcp/orchestrators/vertex_scheduler/main.py +0 -91
  1043. zenml/integrations/gcp/orchestrators/vertex_scheduler/requirements.txt +0 -2
  1044. zenml/integrations/kserve/__init__.py +0 -57
  1045. zenml/integrations/kserve/custom_deployer/zenml_custom_model.py +0 -175
  1046. zenml/integrations/kserve/flavors/kserve_model_deployer_flavor.py +0 -137
  1047. zenml/integrations/kserve/model_deployers/kserve_model_deployer.py +0 -1003
  1048. zenml/integrations/kserve/secret_schemas/secret_schemas.py +0 -65
  1049. zenml/integrations/kserve/services/kserve_deployment.py +0 -596
  1050. zenml/integrations/kserve/steps/__init__.py +0 -22
  1051. zenml/integrations/kserve/steps/kserve_deployer.py +0 -472
  1052. zenml/integrations/kserve/steps/kserve_step_utils.py +0 -293
  1053. zenml/integrations/kubeflow/utils.py +0 -96
  1054. zenml/lineage_graph/lineage_graph.py +0 -244
  1055. zenml/lineage_graph/node/artifact_node.py +0 -52
  1056. zenml/lineage_graph/node/step_node.py +0 -41
  1057. zenml/models/v2/base/internal.py +0 -37
  1058. zenml/models/v2/base/update.py +0 -40
  1059. zenml/models/v2/misc/hub_plugin_models.py +0 -79
  1060. zenml/new/pipelines/deserialization_utils.py +0 -291
  1061. zenml/new/pipelines/model_utils.py +0 -72
  1062. zenml/new/pipelines/pipeline_decorator.py +0 -109
  1063. zenml/new/steps/step_decorator.py +0 -160
  1064. zenml/pipelines/base_pipeline.py +0 -274
  1065. zenml/post_execution/pipeline.py +0 -58
  1066. zenml/post_execution/pipeline_run.py +0 -55
  1067. zenml/services/service_registry.py +0 -214
  1068. zenml/services/terraform/__init__.py +0 -14
  1069. zenml/services/terraform/terraform_service.py +0 -441
  1070. zenml/steps/step_environment.py +0 -108
  1071. zenml/steps/step_output.py +0 -36
  1072. zenml/utils/mlstacks_utils.py +0 -635
  1073. zenml/utils/terraform_utils.py +0 -42
  1074. zenml/zen_server/dashboard/_redirects +0 -1
  1075. zenml/zen_server/dashboard/asset-manifest.json +0 -131
  1076. zenml/zen_server/dashboard/manifest.json +0 -25
  1077. zenml/zen_server/dashboard/precache-manifest.c139638dcc4d9d3425353266447a2fad.js +0 -462
  1078. zenml/zen_server/dashboard/robots.txt +0 -2
  1079. zenml/zen_server/dashboard/service-worker.js +0 -39
  1080. zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css +0 -16
  1081. zenml/zen_server/dashboard/static/css/2.5b37d44a.chunk.css.map +0 -1
  1082. zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css +0 -2
  1083. zenml/zen_server/dashboard/static/css/main.77e46c35.chunk.css.map +0 -1
  1084. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js +0 -3
  1085. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.LICENSE.txt +0 -95
  1086. zenml/zen_server/dashboard/static/js/2.bb4cef22.chunk.js.map +0 -1
  1087. zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js +0 -2
  1088. zenml/zen_server/dashboard/static/js/main.270838b4.chunk.js.map +0 -1
  1089. zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js +0 -2
  1090. zenml/zen_server/dashboard/static/js/runtime-main.bfca2edd.js.map +0 -1
  1091. zenml/zen_server/dashboard/static/media/AlertTriangle.28aee535.svg +0 -5
  1092. zenml/zen_server/dashboard/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
  1093. zenml/zen_server/dashboard/static/media/Back.86c23a22.svg +0 -4
  1094. zenml/zen_server/dashboard/static/media/BookOpen.5cb101ff.svg +0 -4
  1095. zenml/zen_server/dashboard/static/media/BoundingBox.1eb98717.svg +0 -10
  1096. zenml/zen_server/dashboard/static/media/Burger.9b1c67d7.svg +0 -3
  1097. zenml/zen_server/dashboard/static/media/Cached.2381fb8d.svg +0 -1
  1098. zenml/zen_server/dashboard/static/media/Calendar.356e11c7.svg +0 -3
  1099. zenml/zen_server/dashboard/static/media/ChartBarHorizontal.0247447b.svg +0 -6
  1100. zenml/zen_server/dashboard/static/media/ChartLine.0d79e18d.svg +0 -4
  1101. zenml/zen_server/dashboard/static/media/ChatDots.2e1c9211.svg +0 -6
  1102. zenml/zen_server/dashboard/static/media/Check.dad6beb2.svg +0 -3
  1103. zenml/zen_server/dashboard/static/media/CheckCircleFilled.c19566d0.svg +0 -3
  1104. zenml/zen_server/dashboard/static/media/Checkbox.af50e31e.svg +0 -3
  1105. zenml/zen_server/dashboard/static/media/ChevronDown.f860ce32.svg +0 -3
  1106. zenml/zen_server/dashboard/static/media/ChevronDownLight.6642d756.svg +0 -3
  1107. zenml/zen_server/dashboard/static/media/ChevronLeft.f6edfcdb.svg +0 -3
  1108. zenml/zen_server/dashboard/static/media/CircleCheck.f98fd6ca.svg +0 -1
  1109. zenml/zen_server/dashboard/static/media/Clock.ffc9de95.svg +0 -3
  1110. zenml/zen_server/dashboard/static/media/Close.74e9efbc.svg +0 -5
  1111. zenml/zen_server/dashboard/static/media/CloseWithBorder.6960930a.svg +0 -3
  1112. zenml/zen_server/dashboard/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
  1113. zenml/zen_server/dashboard/static/media/CloudArrowUp.0aecb235.svg +0 -6
  1114. zenml/zen_server/dashboard/static/media/Code.ef0f33b5.svg +0 -3
  1115. zenml/zen_server/dashboard/static/media/Config.0be63f8a.svg +0 -1
  1116. zenml/zen_server/dashboard/static/media/Connector.9fd46ef1.svg +0 -10
  1117. zenml/zen_server/dashboard/static/media/Copy.36e2112a.svg +0 -1
  1118. zenml/zen_server/dashboard/static/media/Dashboard.d05787e0.svg +0 -3
  1119. zenml/zen_server/dashboard/static/media/Data.b1c3b5f8.svg +0 -3
  1120. zenml/zen_server/dashboard/static/media/Delete.3c361b28.svg +0 -8
  1121. zenml/zen_server/dashboard/static/media/Docs.7541d478.svg +0 -7
  1122. zenml/zen_server/dashboard/static/media/Download.fba04d87.svg +0 -5
  1123. zenml/zen_server/dashboard/static/media/Edit.490eb294.svg +0 -6
  1124. zenml/zen_server/dashboard/static/media/EmptyRightArrow.23749d01.svg +0 -3
  1125. zenml/zen_server/dashboard/static/media/Example.6396cd37.svg +0 -5
  1126. zenml/zen_server/dashboard/static/media/Extension.1394cd4a.svg +0 -3
  1127. zenml/zen_server/dashboard/static/media/Eye.d9e4ee62.svg +0 -4
  1128. zenml/zen_server/dashboard/static/media/Failed.0213c1a0.svg +0 -1
  1129. zenml/zen_server/dashboard/static/media/FileText.1f15bacd.svg +0 -7
  1130. zenml/zen_server/dashboard/static/media/Filter.ab6b9c0d.svg +0 -3
  1131. zenml/zen_server/dashboard/static/media/Folders.12b29887.svg +0 -5
  1132. zenml/zen_server/dashboard/static/media/FunnelFill.6df4c143.svg +0 -3
  1133. zenml/zen_server/dashboard/static/media/GitCommit.7dd9c2aa.svg +0 -5
  1134. zenml/zen_server/dashboard/static/media/GitHub_Logo.cefc2023.png +0 -0
  1135. zenml/zen_server/dashboard/static/media/Graph.2c63a892.svg +0 -11
  1136. zenml/zen_server/dashboard/static/media/History.08329240.svg +0 -3
  1137. zenml/zen_server/dashboard/static/media/Home.0843b0d5.svg +0 -3
  1138. zenml/zen_server/dashboard/static/media/ImageBuilder.ea762d9c.svg +0 -6
  1139. zenml/zen_server/dashboard/static/media/InProgress.304a0edc.svg +0 -1
  1140. zenml/zen_server/dashboard/static/media/Info.9fe10c5c.svg +0 -3
  1141. zenml/zen_server/dashboard/static/media/KeyboardReturn.491afbe3.svg +0 -3
  1142. zenml/zen_server/dashboard/static/media/Link.72bbb55d.svg +0 -4
  1143. zenml/zen_server/dashboard/static/media/Lock.30f5e1fe.svg +0 -5
  1144. zenml/zen_server/dashboard/static/media/Lock2.a769ea52.svg +0 -3
  1145. zenml/zen_server/dashboard/static/media/LockKey.92f21621.svg +0 -6
  1146. zenml/zen_server/dashboard/static/media/Logs.8bf4d005.svg +0 -5
  1147. zenml/zen_server/dashboard/static/media/MinusCircle.4188f418.svg +0 -4
  1148. zenml/zen_server/dashboard/static/media/ModelRegistry.f0de050a.svg +0 -6
  1149. zenml/zen_server/dashboard/static/media/MultiUser.a2ba7c67.svg +0 -10
  1150. zenml/zen_server/dashboard/static/media/PaginationFirst.92628634.svg +0 -4
  1151. zenml/zen_server/dashboard/static/media/PaginationLast.00d3c732.svg +0 -4
  1152. zenml/zen_server/dashboard/static/media/PaginationNext.86158845.svg +0 -3
  1153. zenml/zen_server/dashboard/static/media/PaginationPrev.60c18a88.svg +0 -3
  1154. zenml/zen_server/dashboard/static/media/Pen.f2d831d4.svg +0 -6
  1155. zenml/zen_server/dashboard/static/media/PhotoCamera.179d6d4c.svg +0 -3
  1156. zenml/zen_server/dashboard/static/media/Pipeline.30d298b0.svg +0 -7
  1157. zenml/zen_server/dashboard/static/media/Plus.5aa1c16b.svg +0 -3
  1158. zenml/zen_server/dashboard/static/media/PlusCircle.92d860dd.svg +0 -5
  1159. zenml/zen_server/dashboard/static/media/Repositories.71a36b8c.svg +0 -3
  1160. zenml/zen_server/dashboard/static/media/RightArrow.f30d3871.svg +0 -29
  1161. zenml/zen_server/dashboard/static/media/Rocket.63bf7b9d.svg +0 -3
  1162. zenml/zen_server/dashboard/static/media/RocketLaunch.1bff2b59.svg +0 -6
  1163. zenml/zen_server/dashboard/static/media/Rubik-Medium.c87313aa.ttf +0 -0
  1164. zenml/zen_server/dashboard/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
  1165. zenml/zen_server/dashboard/static/media/Run.daec4fb2.svg +0 -6
  1166. zenml/zen_server/dashboard/static/media/Search.d1afcce5.svg +0 -4
  1167. zenml/zen_server/dashboard/static/media/Settings.59ca73ae.svg +0 -4
  1168. zenml/zen_server/dashboard/static/media/Share2.46c3ff66.svg +0 -3
  1169. zenml/zen_server/dashboard/static/media/SignOut.6aa718c5.svg +0 -3
  1170. zenml/zen_server/dashboard/static/media/SimplePlus.5cf7ec20.svg +0 -3
  1171. zenml/zen_server/dashboard/static/media/SingleUser.bef3a095.svg +0 -4
  1172. zenml/zen_server/dashboard/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
  1173. zenml/zen_server/dashboard/static/media/Stack.19b604ac.svg +0 -5
  1174. zenml/zen_server/dashboard/static/media/StackComponent.b1ba90b5.svg +0 -4
  1175. zenml/zen_server/dashboard/static/media/Star.f0c25022.svg +0 -9
  1176. zenml/zen_server/dashboard/static/media/StarOutline.94ca8cd9.svg +0 -3
  1177. zenml/zen_server/dashboard/static/media/Storefront.4b4796fe.svg +0 -3
  1178. zenml/zen_server/dashboard/static/media/Stream.543e3039.svg +0 -3
  1179. zenml/zen_server/dashboard/static/media/SupportAgent.510ddf1f.svg +0 -8
  1180. zenml/zen_server/dashboard/static/media/Table.77033750.svg +0 -6
  1181. zenml/zen_server/dashboard/static/media/Tool.d5785486.svg +0 -3
  1182. zenml/zen_server/dashboard/static/media/UserPlus.741a99d7.svg +0 -6
  1183. zenml/zen_server/dashboard/static/media/Verified.0625b2a0.svg +0 -3
  1184. zenml/zen_server/dashboard/static/media/addNew.4fb6c939.svg +0 -8
  1185. zenml/zen_server/dashboard/static/media/arrowClose.cbd53f3f.svg +0 -3
  1186. zenml/zen_server/dashboard/static/media/arrowOpen.6ceef0af.svg +0 -3
  1187. zenml/zen_server/dashboard/static/media/check_small.30bc0138.svg +0 -3
  1188. zenml/zen_server/dashboard/static/media/circleArrowSideClose.98d6013e.svg +0 -18
  1189. zenml/zen_server/dashboard/static/media/circleArrowSideOpen.63653df6.svg +0 -18
  1190. zenml/zen_server/dashboard/static/media/image.104fd14b.png +0 -0
  1191. zenml/zen_server/dashboard/static/media/imageAddIcon.e83004a9.svg +0 -7
  1192. zenml/zen_server/dashboard/static/media/logo.93333e5c.svg +0 -1
  1193. zenml/zen_server/dashboard/static/media/logo_small.4204397d.svg +0 -3
  1194. zenml/zen_server/dashboard/static/media/logo_white.d4b4414e.svg +0 -20
  1195. zenml/zen_server/dashboard/static/media/notConnected.5e2c8ea7.svg +0 -8
  1196. zenml/zen_server/dashboard/static/media/plugin-fallback.72c294e6.svg +0 -6
  1197. zenml/zen_server/dashboard/static/media/share.bcd998b0.svg +0 -5
  1198. zenml/zen_server/dashboard/static/media/stars.08a9b19a.svg +0 -8
  1199. zenml/zen_server/deploy/terraform/__init__.py +0 -41
  1200. zenml/zen_server/deploy/terraform/providers/__init__.py +0 -14
  1201. zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
  1202. zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
  1203. zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
  1204. zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -332
  1205. zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
  1206. zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
  1207. zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
  1208. zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -14
  1209. zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
  1210. zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
  1211. zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
  1212. zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -179
  1213. zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
  1214. zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -111
  1215. zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
  1216. zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
  1217. zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
  1218. zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
  1219. zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -14
  1220. zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
  1221. zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
  1222. zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
  1223. zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
  1224. zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -188
  1225. zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -111
  1226. zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
  1227. zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
  1228. zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
  1229. zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -14
  1230. zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
  1231. zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
  1232. zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
  1233. zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -183
  1234. zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -122
  1235. zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -255
  1236. zenml/zen_server/routers/run_metadata_endpoints.py +0 -97
  1237. zenml_nightly-0.55.0.dev20240124.dist-info/METADATA +0 -438
  1238. zenml_nightly-0.55.0.dev20240124.dist-info/RECORD +0 -1072
  1239. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/LICENSE +0 -0
  1240. {zenml_nightly-0.55.0.dev20240124.dist-info → zenml_nightly-0.72.0.dev20250116.dist-info}/entry_points.txt +0 -0
zenml/cli/__init__.py CHANGED
@@ -80,7 +80,7 @@ You can also pass in a directory path manually using the
80
80
  zenml init --path /path/to/dir
81
81
  ```
82
82
 
83
- If you wish to use one of [the available ZenML project templates](https://docs.zenml.io/user-guide/starter-guide/using-project-templates#list-of-zenml-project-templates)
83
+ If you wish to use one of [the available ZenML project templates](https://docs.zenml.io/how-to/setting-up-a-project-repository/using-project-templates#list-of-zenml-project-templates)
84
84
  to generate a ready-to-use project scaffold in your repository, you can do so by
85
85
  passing the ``--template`` option:
86
86
 
@@ -96,13 +96,23 @@ you can add ``--template-with-defaults`` to the same command, like this:
96
96
  zenml init --template <name_of_template> --template-with-defaults
97
97
  ```
98
98
 
99
+ In a similar fashion, if you would like to quickly explore the capabilities
100
+ of ZenML through a notebook, you can also use:
101
+
102
+ ```bash
103
+ zenml go
104
+ ```
105
+
106
+ Cleaning up
107
+ -----------
108
+
99
109
  If you wish to delete all data relating to your workspace from the
100
110
  directory, use the ``zenml clean`` command. This will:
101
111
 
102
112
  - delete all pipelines, pipeline runs and associated metadata
103
113
  - delete all artifacts
104
114
 
105
- Using integrations
115
+ Using Integrations
106
116
  ------------------
107
117
 
108
118
  Integrations are the different pieces of a project stack that enable custom
@@ -148,8 +158,29 @@ Uninstalling a specific integration is as simple as typing:
148
158
  zenml integration uninstall INTEGRATION_NAME
149
159
  ```
150
160
 
151
- Filtering CLI output when listing
152
- ---------------------------------
161
+ For all these `zenml integration` commands, you can pass the `--uv` flag and we
162
+ will use `uv` as the package manager instead of `pip`. This will resolve and
163
+ install much faster than with `pip`, but note that it requires `uv` to be
164
+ installed on your machine. This is an experimental feature and may not work on
165
+ all systems. In particular, note that installing onto machines with GPU
166
+ acceleration may not work as expected.
167
+
168
+ If you would like to export the requirements of all ZenML integrations, you can
169
+ use the command:
170
+
171
+ ```bash
172
+ zenml integration export-requirements
173
+ ```
174
+
175
+ Here, you can also select a list of integrations and write the result into and
176
+ output file:
177
+
178
+ ```bash
179
+ zenml integration export-requirements gcp kubeflow -o OUTPUT_FILE
180
+ ```
181
+
182
+ Filtering when listing
183
+ ----------------------
153
184
 
154
185
  Certain CLI `list` commands allow you to filter their output. For example, all
155
186
  stack components allow you to pass custom parameters to the `list` command that
@@ -198,13 +229,21 @@ zenml orchestrator list --created "gt:2021-01-01 00:00:00"
198
229
  This syntax can also be combined to create more complex filters using the `or`
199
230
  and `and` keywords.
200
231
 
201
- Customizing your Artifact Store
202
- -------------------------------
232
+ Artifact Stores
233
+ ---------------
234
+
235
+ In ZenML, [the artifact store](https://docs.zenml.io/stack-components/artifact-stores)
236
+ is where all the inputs and outputs of your pipeline steps are stored. By
237
+ default, ZenML initializes your repository with an artifact store with
238
+ everything kept on your local machine. You can get a better understanding
239
+ about the concept of artifact stores by executing:
203
240
 
204
- The artifact store is where all the inputs and outputs of your pipeline
205
- steps are stored. By default, ZenML initializes your repository with an
206
- artifact store with everything kept on your local machine. If you wish
207
- to register a new artifact store, do so with the ``register`` command:
241
+ ```bash
242
+ zenml artifact-store explain
243
+ ```
244
+
245
+ If you wish to register a new artifact store, do so with the ``register``
246
+ command:
208
247
 
209
248
  ```bash
210
249
  zenml artifact-store register ARTIFACT_STORE_NAME --flavor=ARTIFACT_STORE_FLAVOR [--OPTIONS]
@@ -216,13 +255,52 @@ You can also add any labels to your stack component using the `--label` or `-l`
216
255
  zenml artifact-store register ARTIFACT_STORE_NAME --flavor=ARTIFACT_STORE_FLAVOR -l key1=value1 -l key2=value2
217
256
  ```
218
257
 
219
- If you wish to list the artifact stores that have already been
220
- registered within your ZenML workspace / repository, type:
258
+ As you can see from the command above, when you register a new artifact store,
259
+ you have to choose a flavor. To see the full list of available artifact
260
+ store flavors, you can use the command:
261
+
262
+ ```bash
263
+ zenml artifact-store flavor list
264
+ ```
265
+
266
+ This list will show you which integration these flavors belong to and which
267
+ service connectors they are adaptable with. If you would like to get additional
268
+ information regarding a specific flavor, you can utilize the command:
269
+
270
+ ```bash
271
+ zenml artifact-store flavor describe FLAVOR_NAME
272
+ ```
273
+
274
+ If you wish to list the artifact stores that have already been registered
275
+ within your ZenML:
221
276
 
222
277
  ```bash
223
278
  zenml artifact-store list
224
279
  ```
225
280
 
281
+ If you want the name of the artifact store in the active stack, you can
282
+ also use the `get` command:
283
+
284
+ ```bash
285
+ zenml artifact-store get
286
+ ```
287
+
288
+ For details about a particular artifact store, use the `describe` command.
289
+ By default, (without a specific artifact store name passed in) it will describe
290
+ the active or currently used artifact store:
291
+
292
+ ```bash
293
+ zenml artifact-store describe ARTIFACT_STORE_NAME
294
+ ```
295
+
296
+ If you wish to update/rename an artifact store, you can use the following
297
+ commands respectively:
298
+
299
+ ```bash
300
+ zenml artifact-store update ARTIFACT_STORE_NAME --property_to_update=new_value
301
+ zenml artifact-store rename ARTIFACT_STORE_OLD_NAME ARTIFACT_STORE_NEW_NAME
302
+ ```
303
+
226
304
  If you wish to delete a particular artifact store, pass the name of the
227
305
  artifact store into the CLI with the following command:
228
306
 
@@ -230,13 +308,35 @@ artifact store into the CLI with the following command:
230
308
  zenml artifact-store delete ARTIFACT_STORE_NAME
231
309
  ```
232
310
 
233
- Customizing your Orchestrator
234
- -----------------------------
311
+ If you would like to connect/disconnect your artifact store to/from a service
312
+ connector, you can use the following commands:
313
+
314
+ ```bash
315
+ zenml artifact-store connect ARTIFACT_STORE_NAME -c CONNECTOR_NAME
316
+ zenml artifact-store disconnect
317
+ ```
318
+
319
+ The ZenML CLI provides a few more utility functions for you to manage your
320
+ artifact stores. In order to get a full list of available functions, use
321
+ the command:
322
+
323
+ ```bash
324
+ zenml artifact-store --help
325
+ ```
326
+
327
+ Orchestrators
328
+ -------------
329
+
330
+ An [orchestrator](https://docs.zenml.io/stack-components/orchestrators)
331
+ is a special kind of backend that manages the running of each step of the
332
+ pipeline. Orchestrators administer the actual pipeline runs. By default,
333
+ ZenML initializes your repository with an orchestrator that runs everything
334
+ on your local machine. In order to get a more detailed explanation, you can use
335
+ the command:
235
336
 
236
- An orchestrator is a special kind of backend that manages the running of
237
- each step of the pipeline. Orchestrators administer the actual pipeline
238
- runs. By default, ZenML initializes your repository with an orchestrator
239
- that runs everything on your local machine.
337
+ ```bash
338
+ zenml orchestrator explain
339
+ ```
240
340
 
241
341
  If you wish to register a new orchestrator, do so with the ``register``
242
342
  command:
@@ -251,6 +351,22 @@ You can also add any label to your stack component using the `--label` or `-l` f
251
351
  zenml orchestrator register ORCHESTRATOR_NAME --flavor=ORCHESTRATOR_FLAVOR -l key1=value1 -l key2=value2
252
352
  ```
253
353
 
354
+ As you can see from the command above, when you register a new orchestrator,
355
+ you have to choose a flavor. To see the full list of available orchestrator
356
+ flavors, you can use the command:
357
+
358
+ ```bash
359
+ zenml orchestrator flavor list
360
+ ```
361
+
362
+ This list will show you which integration these flavors belong to and which
363
+ service connectors they are adaptable with. If you would like to get additional
364
+ information regarding a specific flavor, you can utilize the command:
365
+
366
+ ```bash
367
+ zenml orchestrator flavor describe FLAVOR_NAME
368
+ ```
369
+
254
370
  If you wish to list the orchestrators that have already been registered
255
371
  within your ZenML workspace / repository, type:
256
372
 
@@ -258,6 +374,29 @@ within your ZenML workspace / repository, type:
258
374
  zenml orchestrator list
259
375
  ```
260
376
 
377
+ If you want the name of the orchestrator in the active stack, you can
378
+ also use the `get` command:
379
+
380
+ ```bash
381
+ zenml orchestrator get
382
+ ```
383
+
384
+ For details about a particular orchestrator, use the `describe` command.
385
+ By default, (without a specific orchestrator name passed in) it will describe
386
+ the active or currently used orchestrator:
387
+
388
+ ```bash
389
+ zenml orchestrator describe [ORCHESTRATOR_NAME]
390
+ ```
391
+
392
+ If you wish to update/rename an orchestrator, you can use the following
393
+ commands respectively:
394
+
395
+ ```bash
396
+ zenml orchestrator update ORCHESTRATOR_NAME --property_to_update=new_value
397
+ zenml orchestrator rename ORCHESTRATOR_OLD_NAME ORCHESTRATOR_NEW_NAME
398
+ ```
399
+
261
400
  If you wish to delete a particular orchestrator, pass the name of the
262
401
  orchestrator into the CLI with the following command:
263
402
 
@@ -265,13 +404,37 @@ orchestrator into the CLI with the following command:
265
404
  zenml orchestrator delete ORCHESTRATOR_NAME
266
405
  ```
267
406
 
268
- Customizing your Container Registry
269
- -----------------------------------
407
+ If you would like to connect/disconnect your orchestrator to/from a service
408
+ connector, you can use the following commands:
409
+
410
+ ```bash
411
+ zenml orchestrator connect ORCHESTRATOR_NAME -c CONNECTOR_NAME
412
+ zenml orchestrator disconnect
413
+ ```
414
+
415
+ The ZenML CLI provides a few more utility functions for you to manage your
416
+ orchestrators. In order to get a full list of available functions, use
417
+ the command:
418
+
419
+ ```bash
420
+ zenml orchestrators --help
421
+ ```
270
422
 
271
- The container registry is where all the images that are used by a
272
- container-based orchestrator are stored. By default, a default ZenML local stack
273
- will not register a container registry. If you wish to register a new container
274
- registry, do so with the `register` command:
423
+ Container Registries
424
+ --------------------
425
+
426
+ [The container registry](https://docs.zenml.io/stack-components/container-registries)
427
+ is where all the images that are used by a container-based orchestrator are
428
+ stored. To get a better understanding regarding container registries, use
429
+ the command:
430
+
431
+ ```bash
432
+ zenml container-registry explain
433
+ ```
434
+
435
+ By default, a default ZenML local stack will not register a container registry.
436
+ If you wish to register a new container registry, do so with the `register`
437
+ command:
275
438
 
276
439
  ```bash
277
440
  zenml container-registry register REGISTRY_NAME --flavor=REGISTRY_FLAVOR [--REGISTRY_OPTIONS]
@@ -283,10 +446,20 @@ You can also add any label to your stack component using the `--label` or `-l` f
283
446
  zenml container-registry register REGISTRY_NAME --flavor=REGISTRY_FLAVOR -l key1=value1 -l key2=value2
284
447
  ```
285
448
 
286
- If you want the name of the current container registry, use the `get` command:
449
+ As you can see from the command above, when you register a new container
450
+ registry, you have to choose a flavor. To see the full list of available
451
+ container registry flavors, you can use the command:
287
452
 
288
453
  ```bash
289
- zenml container-registry get
454
+ zenml container-registry flavor list
455
+ ```
456
+
457
+ This list will show you which integration these flavors belong to and which
458
+ service connectors they are adaptable with. If you would like to get additional
459
+ information regarding a specific flavor, you can utilize the command:
460
+
461
+ ```bash
462
+ zenml container-registry flavor describe FLAVOR_NAME
290
463
  ```
291
464
 
292
465
  To list all container registries available and registered for use, use the
@@ -296,12 +469,27 @@ To list all container registries available and registered for use, use the
296
469
  zenml container-registry list
297
470
  ```
298
471
 
472
+ If you want the name of the container registry in the active stack, you can
473
+ also use the `get` command:
474
+
475
+ ```bash
476
+ zenml container-registry get
477
+ ```
478
+
299
479
  For details about a particular container registry, use the `describe` command.
300
480
  By default, (without a specific registry name passed in) it will describe the
301
481
  active or currently used container registry:
302
482
 
303
483
  ```bash
304
- zenml container-registry describe [REGISTRY_NAME]
484
+ zenml container-registry describe [CONTAINER_REGISTRY_NAME]
485
+ ```
486
+
487
+ If you wish to update/rename a container registry, you can use the following
488
+ commands respectively:
489
+
490
+ ```bash
491
+ zenml container-registry update CONTAINER_REGISTRY_NAME --property_to_update=new_value
492
+ zenml container-registry rename CONTAINER_REGISTRY_OLD_NAME CONTAINER_REGISTRY_NEW_NAME
305
493
  ```
306
494
 
307
495
  To delete a container registry (and all of its contents), use the `delete`
@@ -311,275 +499,304 @@ command:
311
499
  zenml container-registry delete REGISTRY_NAME
312
500
  ```
313
501
 
314
- Customizing your Experiment Tracker
315
- -----------------------------------
316
-
317
- Experiment trackers let you track your ML experiments by logging the parameters
318
- and allowing you to compare between different runs. If you want to use an
319
- experiment tracker in one of your stacks, you need to first register it:
502
+ If you would like to connect/disconnect your container registry to/from a
503
+ service connector, you can use the following commands:
320
504
 
321
505
  ```bash
322
- zenml experiment-tracker register EXPERIMENT_TRACKER_NAME \
323
- --flavor=EXPERIMENT_TRACKER_FLAVOR [--EXPERIMENT_TRACKER_OPTIONS]
506
+ zenml container-registry connect CONTAINER_REGISTRY_NAME -c CONNECTOR_NAME
507
+ zenml container-registry disconnect
324
508
  ```
325
509
 
326
- You can also add any label to your stack component using the `--label` or `-l` flag:
510
+ The ZenML CLI provides a few more utility functions for you to manage your
511
+ container registries. In order to get a full list of available functions,
512
+ use the command:
327
513
 
328
514
  ```bash
329
- zenml experiment-tracker register EXPERIMENT_TRACKER_NAME \
330
- --flavor=EXPERIMENT_TRACKER_FLAVOR -l key1=value1 -l key2=value2
515
+ zenml container-registry --help
331
516
  ```
332
517
 
333
- If you want the name of the current experiment tracker, use the `get` command:
518
+ Data Validators
519
+ ---------------
520
+
521
+ In ZenML, [data validators](https://docs.zenml.io/stack-components/data-validators)
522
+ help you profile and validate your data.
523
+
524
+ By default, a default ZenML local stack will not register a data validator. If
525
+ you wish to register a new data validator, do so with the `register` command:
334
526
 
335
527
  ```bash
336
- zenml experiment-tracker get
528
+ zenml data-validator register DATA_VALIDATOR_NAME --flavor DATA_VALIDATOR_FLAVOR [--DATA_VALIDATOR_OPTIONS]
337
529
  ```
338
530
 
339
- To list all experiment trackers available and registered for use, use the
340
- `list` command:
531
+ You can also add any label to your stack component using the `--label` or `-l` flag:
341
532
 
342
533
  ```bash
343
- zenml experiment-tracker list
534
+ zenml data-validator register DATA_VALIDATOR_NAME --flavor DATA_VALIDATOR_FLAVOR -l key1=value1 -l key2=value2
344
535
  ```
345
536
 
346
- For details about a particular experiment tracker, use the `describe` command.
347
- By default, (without a specific experiment tracker name passed in) it will
348
- describe the active or currently-used experiment tracker:
537
+ As you can see from the command above, when you register a new data validator,
538
+ you have to choose a flavor. To see the full list of available data validator
539
+ flavors, you can use the command:
349
540
 
350
541
  ```bash
351
- zenml experiment-tracker describe [EXPERIMENT_TRACKER_NAME]
542
+ zenml data-validator flavor list
352
543
  ```
353
544
 
354
- To delete an experiment tracker, use the `delete` command:
545
+ This list will show you which integration these flavors belong to and which
546
+ service connectors they are adaptable with. If you would like to get additional
547
+ information regarding a specific flavor, you can utilize the command:
355
548
 
356
549
  ```bash
357
- zenml experiment-tracker delete EXPERIMENT_TRACKER_NAME
550
+ zenml data-validator flavor describe FLAVOR_NAME
358
551
  ```
359
552
 
360
- Customizing your Step Operator
361
- ------------------------------
553
+ To list all data validators available and registered for use, use the `list`
554
+ command:
362
555
 
363
- Step operators allow you to run individual steps in a custom environment
364
- different from the default one used by your active orchestrator. One example
365
- use-case is to run a training step of your pipeline in an environment with GPUs
366
- available. By default, a default ZenML local stack will not register a step
367
- operator. If you wish to register a new step operator, do so with the
368
- `register` command:
556
+ ```bash
557
+ zenml data-validator list
558
+ ```
559
+
560
+ If you want the name of the data validator in the active stack, use the `get`
561
+ command:
369
562
 
370
563
  ```bash
371
- zenml step-operator register STEP_OPERATOR_NAME --flavor STEP_OPERATOR_FLAVOR [--STEP_OPERATOR_OPTIONS]
564
+ zenml data-validator get
372
565
  ```
373
566
 
374
- You can also add any label to your stack component using the `--label` or `-l` flag:
567
+ For details about a particular data validator, use the `describe` command.
568
+ By default, (without a specific data validator name passed in) it will describe
569
+ the active or currently-used data validator:
375
570
 
376
571
  ```bash
377
- zenml step-operator register STEP_OPERATOR_NAME --flavor STEP_OPERATOR_FLAVOR -l key1=value1 -l key2=value2
572
+ zenml data-validator describe [DATA_VALIDATOR_NAME]
378
573
  ```
379
574
 
380
- If you want the name of the current step operator, use the `get` command:
575
+ If you wish to update/rename a data validator, you can use the following
576
+ commands respectively:
381
577
 
382
578
  ```bash
383
- zenml step-operator get
579
+ zenml data-validator update DATA_VALIDATOR_NAME --property_to_update=new_value
580
+ zenml data-validator rename DATA_VALIDATOR_OLD_NAME DATA_VALIDATOR_NEW_NAME
384
581
  ```
385
582
 
386
- To list all step operators available and registered for use, use the
387
- `list` command:
583
+ To delete a data validator (and all of its contents), use the `delete` command:
388
584
 
389
585
  ```bash
390
- zenml step-operator list
586
+ zenml data-validator delete DATA_VALIDATOR_NAME
391
587
  ```
392
588
 
393
- For details about a particular step operator, use the `describe` command.
394
- By default, (without a specific operator name passed in) it will describe the
395
- active or currently used step operator:
589
+ If you would like to connect/disconnect your data validator to/from a service
590
+ connector, you can use the following commands:
396
591
 
397
592
  ```bash
398
- zenml step-operator describe [STEP_OPERATOR_NAME]
593
+ zenml data-validator connect DATA_VALIDATOR_NAME -c CONNECTOR_NAME
594
+ zenml data-validator disconnect
399
595
  ```
400
596
 
401
- To delete a step operator (and all of its contents), use the `delete`
597
+ The ZenML CLI provides a few more utility functions for you to manage your
598
+ data validators. In order to get a full list of available functions, use the
402
599
  command:
403
600
 
404
601
  ```bash
405
- zenml step-operator delete STEP_OPERATOR_NAME
602
+ zenml data-validator --help
406
603
  ```
407
604
 
408
- Deploying Stack Components
409
- --------------------------
605
+ Experiment Trackers
606
+ -------------------
410
607
 
411
- Stack components can be deployed directly via the CLI. You can use the `deploy`
412
- subcommand for this. For example, you could deploy a GCP artifact store using
413
- the following command:
608
+ [Experiment trackers](https://docs.zenml.io/stack-components/experiment-trackers)
609
+ let you track your ML experiments by logging the parameters
610
+ and allow you to compare between different runs. To get a better
611
+ understanding regarding experiment trackers, use the command:
414
612
 
415
- ```shell
416
- zenml artifact-store deploy -f gcp -p gcp -r us-east1 -x project_id=zenml-core basic_gcp_artifact_store
613
+ ```bash
614
+ zenml experiment-tracker explain
417
615
  ```
418
616
 
419
- For full documentation on this functionality, please refer to [the dedicated
420
- documentation on stack component deploy](https://docs.zenml.io/stacks-and-components/stack-deployment/deploy-a-stack-component).
421
-
422
- Secrets Management
423
- ------------------
617
+ By default, a default ZenML local stack will not register an experiment tracker.
618
+ If you want to use an experiment tracker in one of your stacks, you need to
619
+ first register it:
424
620
 
425
- ZenML offers a way to securely store secrets associated with your other
426
- stack components and infrastructure. A ZenML Secret is a collection or grouping
427
- of key-value pairs stored by the ZenML secrets store.
428
- ZenML Secrets are identified by a unique name which allows you to fetch or
429
- reference them in your pipelines and stacks.
621
+ ```bash
622
+ zenml experiment-tracker register EXPERIMENT_TRACKER_NAME \
623
+ --flavor=EXPERIMENT_TRACKER_FLAVOR [--EXPERIMENT_TRACKER_OPTIONS]
624
+ ```
430
625
 
431
- Depending on how you set up and deployed ZenML, the secrets store keeps secrets
432
- in the local database or uses the ZenML server your client is connected to:
626
+ You can also add any label to your stack component using the `--label` or `-l` flag:
433
627
 
434
- * if you are using the default ZenML client settings, or if you connect your
435
- ZenML client to a local ZenML server started with `zenml up`, the secrets store
436
- is using the same local SQLite database as the rest of ZenML
437
- * if you connect your ZenML client to a remote ZenML server, the
438
- secrets are no longer managed on your local machine, but through the remote
439
- server instead. Secrets are stored in whatever secrets store back-end the
440
- remote server is configured to use. This can be a SQL database, one of the
441
- managed cloud secrets management services, or even a custom back-end.
628
+ ```bash
629
+ zenml experiment-tracker register EXPERIMENT_TRACKER_NAME \
630
+ --flavor=EXPERIMENT_TRACKER_FLAVOR -l key1=value1 -l key2=value2
631
+ ```
442
632
 
443
- To create a secret, use the `create` command and pass the key-value pairs
444
- as command-line arguments:
633
+ As you can see from the command above, when you register a new experiment
634
+ tracker, you have to choose a flavor. To see the full list of available
635
+ experiment tracker flavors, you can use the command:
445
636
 
446
637
  ```bash
447
- zenml secret create SECRET_NAME --key1=value1 --key2=value2 --key3=value3 ...
638
+ zenml experiment-tracker flavor list
639
+ ```
448
640
 
449
- # Another option is to use the '--values' option and provide key-value pairs in either JSON or YAML format.
450
- zenml secret create SECRET_NAME --values='{"key1":"value2","key2":"value2","key3":"value3"}'
641
+ This list will show you which integration these flavors belong to and which
642
+ service connectors they are adaptable with. If you would like to get additional
643
+ information regarding a specific flavor, you can utilize the command:
644
+
645
+ ```bash
646
+ zenml experiment-tracker flavor describe FLAVOR_NAME
451
647
  ```
452
648
 
453
- Note that when using the previous command the keys and values will be preserved in your `bash_history` file, so
454
- you may prefer to use the interactive `create` command instead:
649
+ To list all experiment trackers available and registered for use, use the
650
+ `list` command:
455
651
 
456
- ```shell
457
- zenml secret create SECRET_NAME -i
652
+ ```bash
653
+ zenml experiment-tracker list
458
654
  ```
459
655
 
460
- As an alternative to the interactive mode, also useful for values that
461
- are long or contain newline or special characters, you can also use the special
462
- `@` syntax to indicate to ZenML that the value needs to be read from a file:
656
+ If you want the name of the experiment tracker in the active stack, use the
657
+ `get` command:
463
658
 
464
659
  ```bash
465
- zenml secret create SECRET_NAME \
466
- --aws_access_key_id=1234567890 \
467
- --aws_secret_access_key=abcdefghij \
468
- --aws_session_token=@/path/to/token.txt
660
+ zenml experiment-tracker get
661
+ ```
469
662
 
470
- # Alternatively for providing key-value pairs, you can utilize the '--values' option by specifying a file path containing
471
- # key-value pairs in either JSON or YAML format.
472
- zenml secret create SECRET_NAME --values=@/path/to/token.txt
663
+ For details about a particular experiment tracker, use the `describe` command.
664
+ By default, (without a specific experiment tracker name passed in) it will
665
+ describe the active or currently-used experiment tracker:
666
+
667
+ ```bash
668
+ zenml experiment-tracker describe [EXPERIMENT_TRACKER_NAME]
473
669
  ```
474
670
 
475
- To list all the secrets available, use the `list` command:
671
+ If you wish to update/rename an experiment tracker, you can use the following
672
+ commands respectively:
476
673
 
477
674
  ```bash
478
- zenml secret list
675
+ zenml experiment-tracker update EXPERIMENT_TRACKER_NAME --property_to_update=new_value
676
+ zenml experiment-tracker rename EXPERIMENT_TRACKER_OLD_NAME EXPERIMENT_TRACKER_NEW_NAME
479
677
  ```
480
678
 
481
- To get the key-value pairs for a particular secret, use the `get` command:
679
+ To delete an experiment tracker, use the `delete` command:
482
680
 
483
681
  ```bash
484
- zenml secret get SECRET_NAME
682
+ zenml experiment-tracker delete EXPERIMENT_TRACKER_NAME
485
683
  ```
486
684
 
487
- To update a secret, use the `update` command:
685
+ If you would like to connect/disconnect your experiment tracker to/from a
686
+ service connector, you can use the following commands:
488
687
 
489
688
  ```bash
490
- zenml secret update SECRET_NAME --key1=value1 --key2=value2 --key3=value3 ...
689
+ zenml experiment-tracker connect EXPERIMENT_TRACKER_NAME -c CONNECTOR_NAME
690
+ zenml experiment-tracker disconnect
691
+ ```
491
692
 
492
- # Another option is to use the '--values' option and provide key-value pairs in either JSON or YAML format.
493
- zenml secret update SECRET_NAME --values='{"key1":"value2","key2":"value2","key3":"value3"}'
693
+ The ZenML CLI provides a few more utility functions for you to manage your
694
+ experiment trackers. In order to get a full list of available functions,
695
+ use the command:
696
+
697
+ ```bash
698
+ zenml experiment-tracker --help
494
699
  ```
495
700
 
496
- Note that when using the previous command the keys and values will be preserved in your `bash_history` file, so
497
- you may prefer to use the interactive `update` command instead:
701
+ Model Deployers
702
+ ---------------
498
703
 
499
- ```shell
500
- zenml secret update SECRET_NAME -i
704
+ [Model deployers](https://docs.zenml.io/stack-components/model-deployers)
705
+ are stack components responsible for online model serving. They are responsible
706
+ for deploying models to a remote server. Model deployers also act as a registry
707
+ for models that are served with ZenML. To get a better understanding regarding
708
+ model deployers, use the command:
709
+
710
+ ```bash
711
+ zenml model-deployer explain
501
712
  ```
502
713
 
503
- Finally, to delete a secret, use the `delete` command:
714
+ By default, a default ZenML local stack will not register a model deployer. If
715
+ you wish to register a new model deployer, do so with the `register` command:
504
716
 
505
717
  ```bash
506
- zenml secret delete SECRET_NAME
718
+ zenml model-deployer register MODEL_DEPLOYER_NAME --flavor MODEL_DEPLOYER_FLAVOR [--MODEL_DEPLOYER_OPTIONS]
507
719
  ```
508
720
 
509
- Secrets can be scoped to a workspace or a user. By default, secrets
510
- are scoped to the current workspace. To scope a secret to a user, use the
511
- `--scope user` argument in the `register` command.
721
+ You can also add any label to your stack component using the `--label` or `-l` flag:
512
722
 
513
- Add a Feature Store to your Stack
514
- ---------------------------------
723
+ ```bash
724
+ zenml model-deployer register MODEL_DEPLOYER_NAME --flavor MODEL_DEPLOYER_FLAVOR -l key1=value1 -l key2=value2
725
+ ```
515
726
 
516
- ZenML supports connecting to a Redis-backed Feast feature store as a stack
517
- component integration. To set up a feature store, use the following CLI command:
727
+ As you can see from the command above, when you register a new model deployer,
728
+ you have to choose a flavor. To see the full list of available model deployer
729
+ flavors, you can use the command:
518
730
 
519
- ```shell
520
- zenml feature-store register FEATURE_STORE_NAME --flavor=feast
521
- --feast_repo=REPO_PATH --online_host HOST_NAME --online_port ONLINE_PORT_NUMBER
731
+ ```bash
732
+ zenml model-deployer flavor list
522
733
  ```
523
734
 
524
- Once you have registered your feature store as a stack component, you can use it
525
- in your ZenML Stack.
735
+ This list will show you which integration these flavors belong to and which
736
+ service connectors they are adaptable with. If you would like to get additional
737
+ information regarding a specific flavor, you can utilize the command:
526
738
 
527
- Interacting with Model Deployers
528
- --------------------------------
739
+ ```bash
740
+ zenml model-deployer flavor describe FLAVOR_NAME
741
+ ```
529
742
 
530
- Model deployers are stack components responsible for online model serving.
531
- They are responsible for deploying models to a remote server. Model deployers
532
- also act as a registry for models that are served with ZenML.
743
+ To list all model deployers available and registered for use, use the
744
+ `list` command:
533
745
 
534
- If you wish to register a new model deployer, do so with the
535
- `register` command:
746
+ ```bash
747
+ zenml model-deployer list
748
+ ```
749
+
750
+ If you want the name of the model deployer in the active stack, use the `get`
751
+ command:
536
752
 
537
753
  ```bash
538
- zenml model-deployer register MODEL_DEPLOYER_NAME --flavor=MODEL_DEPLOYER_FLAVOR [--OPTIONS]
754
+ zenml model-deployer get
539
755
  ```
540
756
 
541
- If you wish to list the model-deployers that have already been registered
542
- within your ZenML workspace / repository, type:
757
+ For details about a particular model deployer, use the `describe` command.
758
+ By default, (without a specific operator name passed in) it will describe the
759
+ active or currently used model deployer:
543
760
 
544
761
  ```bash
545
- zenml model-deployer list
762
+ zenml model-deployer describe [MODEL_DEPLOYER_NAME]
546
763
  ```
547
764
 
548
- If you wish to get more detailed information about a particular model deployer
549
- within your ZenML workspace / repository, type:
765
+ If you wish to update/rename a model deployer, you can use the following
766
+ commands respectively:
550
767
 
551
768
  ```bash
552
- zenml model-deployer describe MODEL_DEPLOYER_NAME
769
+ zenml model-deployer update MODEL_DEPLOYER_NAME --property_to_update=new_value
770
+ zenml model-deployer rename MODEL_DEPLOYER_OLD_NAME MODEL_DEPLOYER_NEW_NAME
553
771
  ```
554
772
 
555
- If you wish to delete a particular model deployer, pass the name of the
556
- model deployers into the CLI with the following command:
773
+ To delete a model deployer (and all of its contents), use the `delete`
774
+ command:
557
775
 
558
776
  ```bash
559
777
  zenml model-deployer delete MODEL_DEPLOYER_NAME
560
778
  ```
561
779
 
562
- If you wish to retrieve logs corresponding to a particular model deployer, pass
563
- the name of the model deployer into the CLI with the following command:
780
+ If you would like to connect/disconnect your model deployer to/from a
781
+ service connector, you can use the following commands:
564
782
 
565
783
  ```bash
566
- zenml model-deployer logs MODEL_DEPLOYER_NAME
784
+ zenml model-deployer connect MODEL_DEPLOYER_NAME -c CONNECTOR_NAME
785
+ zenml model-deployer disconnect
567
786
  ```
568
787
 
569
- Interacting with Deployed Models
570
- --------------------------------
571
-
572
- If you want to simply see what models have been deployed within your stack, run
573
- the following command:
788
+ Moreover, ZenML features a set of CLI commands specific to the model deployer
789
+ interface. If you want to simply see what models have been deployed within
790
+ your stack, run the following command:
574
791
 
575
792
  ```bash
576
793
  zenml model-deployer models list
577
794
  ```
578
795
 
579
- This should give you a list of served models containing their uuid, the name
796
+ This should give you a list of served models containing their `uuid`, the name
580
797
  of the pipeline that produced them including the run id and the step name as
581
- well as the status.
582
- This information should help you identify the different models.
798
+ well as the status. This information should help you identify the different
799
+ models.
583
800
 
584
801
  If you want further information about a specific model, simply copy the
585
802
  UUID and the following command.
@@ -588,7 +805,7 @@ UUID and the following command.
588
805
  zenml model-deployer models describe <UUID>
589
806
  ```
590
807
 
591
- If you are only interested in the prediction-url of the specific model you can
808
+ If you are only interested in the prediction url of the specific model you can
592
809
  also run:
593
810
 
594
811
  ```bash
@@ -610,29 +827,618 @@ If you want to completely remove a served model you can also irreversibly delete
610
827
  zenml model-deployer models delete <UUID>
611
828
  ```
612
829
 
613
- Administering the Stack
614
- -----------------------
830
+ The ZenML CLI provides a few more utility functions for you to manage your
831
+ model deployers. In order to get a full list of available functions,
832
+ use the command:
615
833
 
616
- The stack is a grouping of your artifact store, your orchestrator, and other
617
- optional MLOps tools like experiment trackers or model deployers.
618
- With the ZenML tool, switching from a local stack to a distributed cloud
619
- environment can be accomplished with just a few CLI commands.
834
+ ```bash
835
+ zenml model-deployer --help
836
+ ```
620
837
 
621
- To register a new stack, you must already have registered the individual
622
- components of the stack using the commands listed above.
838
+ Step Operators
839
+ --------------
623
840
 
624
- Use the ``zenml stack register`` command to register your stack. It
625
- takes four arguments as in the following example:
841
+ [Step operators](https://docs.zenml.io/stack-components/step-operators)
842
+ allow you to run individual steps in a custom environment different from the
843
+ default one used by your active orchestrator. One example use-case is to run a
844
+ training step of your pipeline in an environment with GPUs available. To get
845
+ a better understanding regarding step operators, use the command:
626
846
 
627
847
  ```bash
628
- zenml stack register STACK_NAME \
629
- -a ARTIFACT_STORE_NAME \
630
- -o ORCHESTRATOR_NAME
848
+ zenml step-operator explain
849
+ ```
850
+
851
+ By default, a default ZenML local stack will not register a step operator. If
852
+ you wish to register a new step operator, do so with the `register` command:
853
+
854
+ ```bash
855
+ zenml step-operator register STEP_OPERATOR_NAME --flavor STEP_OPERATOR_FLAVOR [--STEP_OPERATOR_OPTIONS]
856
+ ```
857
+
858
+ You can also add any label to your stack component using the `--label` or `-l` flag:
859
+
860
+ ```bash
861
+ zenml step-operator register STEP_OPERATOR_NAME --flavor STEP_OPERATOR_FLAVOR -l key1=value1 -l key2=value2
862
+ ```
863
+
864
+ As you can see from the command above, when you register a new step operator,
865
+ you have to choose a flavor. To see the full list of available step operator
866
+ flavors, you can use the command:
867
+
868
+ ```bash
869
+ zenml step-operator flavor list
870
+ ```
871
+
872
+ This list will show you which integration these flavors belong to and which
873
+ service connectors they are adaptable with. If you would like to get additional
874
+ information regarding a specific flavor, you can utilize the command:
875
+
876
+ ```bash
877
+ zenml step-operator flavor describe FLAVOR_NAME
878
+ ```
879
+
880
+ To list all step operators available and registered for use, use the
881
+ `list` command:
882
+
883
+ ```bash
884
+ zenml step-operator list
885
+ ```
886
+
887
+ If you want the name of the step operator in the active stack, use the `get`
888
+ command:
889
+
890
+ ```bash
891
+ zenml step-operator get
892
+ ```
893
+
894
+ For details about a particular step operator, use the `describe` command.
895
+ By default, (without a specific operator name passed in) it will describe the
896
+ active or currently used step operator:
897
+
898
+ ```bash
899
+ zenml step-operator describe [STEP_OPERATOR_NAME]
900
+ ```
901
+
902
+ If you wish to update/rename a step operator, you can use the following commands
903
+ respectively:
904
+
905
+ ```bash
906
+ zenml step-operator update STEP_OPERATOR_NAME --property_to_update=new_value
907
+ zenml step-operator rename STEP_OPERATOR_OLD_NAME STEP_OPERATOR_NEW_NAME
908
+ ```
909
+
910
+ To delete a step operator (and all of its contents), use the `delete`
911
+ command:
912
+
913
+ ```bash
914
+ zenml step-operator delete STEP_OPERATOR_NAME
915
+ ```
916
+
917
+ If you would like to connect/disconnect your step operator to/from a
918
+ service connector, you can use the following commands:
919
+
920
+ ```bash
921
+ zenml step-operator connect STEP_OPERATOR_NAME -c CONNECTOR_NAME
922
+ zenml step-operator disconnect
923
+ ```
924
+
925
+ The ZenML CLI provides a few more utility functions for you to manage your
926
+ step operators. In order to get a full list of available functions,
927
+ use the command:
928
+
929
+ ```bash
930
+ zenml step-operator --help
931
+ ```
932
+
933
+ Alerters
934
+ --------
935
+
936
+ In ZenML, [alerters](https://docs.zenml.io/stack-components/alerters)
937
+ allow you to send alerts from within your pipeline.
938
+
939
+ By default, a default ZenML local stack will not register an alerter. If
940
+ you wish to register a new alerter, do so with the `register` command:
941
+
942
+ ```bash
943
+ zenml alerter register ALERTER_NAME --flavor ALERTER_FLAVOR [--ALERTER_OPTIONS]
944
+ ```
945
+
946
+ You can also add any label to your stack component using the `--label` or `-l` flag:
947
+
948
+ ```bash
949
+ zenml alerter register ALERTER_NAME --flavor ALERTER_FLAVOR -l key1=value1 -l key2=value2
950
+ ```
951
+
952
+ As you can see from the command above, when you register a new alerter,
953
+ you have to choose a flavor. To see the full list of available alerter
954
+ flavors, you can use the command:
955
+
956
+ ```bash
957
+ zenml alerter flavor list
958
+ ```
959
+
960
+ This list will show you which integration these flavors belong to and which
961
+ service connectors they are adaptable with. If you would like to get additional
962
+ information regarding a specific flavor, you can utilize the command:
963
+
964
+ ```bash
965
+ zenml alerter flavor describe FLAVOR_NAME
966
+ ```
967
+
968
+ To list all alerters available and registered for use, use the `list` command:
969
+
970
+ ```bash
971
+ zenml alerter list
972
+ ```
973
+
974
+ If you want the name of the alerter in the active stack, use the `get`
975
+ command:
976
+
977
+ ```bash
978
+ zenml alerter get
979
+ ```
980
+
981
+ For details about a particular alerter, use the `describe` command.
982
+ By default, (without a specific alerter name passed in) it will describe
983
+ the active or currently used alerter:
984
+
985
+ ```bash
986
+ zenml alerter describe [ALERTER_NAME]
987
+ ```
988
+
989
+ If you wish to update/rename an alerter, you can use the following commands
990
+ respectively:
991
+
992
+ ```bash
993
+ zenml alerter update ALERTER_NAME --property_to_update=new_value
994
+ zenml alerter rename ALERTER_OLD_NAME ALERTER_NEW_NAME
995
+ ```
996
+
997
+ To delete an alerter (and all of its contents), use the `delete` command:
998
+
999
+ ```bash
1000
+ zenml alerter delete ALERTER_NAME
1001
+ ```
1002
+
1003
+ If you would like to connect/disconnect your alerter to/from a service
1004
+ connector, you can use the following commands:
1005
+
1006
+ ```bash
1007
+ zenml alerter connect ALERTER_NAME -c CONNECTOR_NAME
1008
+ zenml alerter disconnect
1009
+ ```
1010
+
1011
+ The ZenML CLI provides a few more utility functions for you to manage your
1012
+ alerters. In order to get a full list of available functions, use the command:
1013
+
1014
+ ```bash
1015
+ zenml alerter --help
1016
+ ```
1017
+
1018
+ Feature Stores
1019
+ --------------
1020
+
1021
+ [Feature stores](https://docs.zenml.io/stack-components/feature-stores)
1022
+ allow data teams to serve data via an offline store and an online low-latency
1023
+ store where data is kept in sync between the two. To get a better understanding
1024
+ regarding feature stores, use the command:
1025
+
1026
+ ```bash
1027
+ zenml feature-store explain
1028
+ ```
1029
+
1030
+ By default, a default ZenML local stack will not register a feature store. If
1031
+ you wish to register a new feature store, do so with the `register` command:
1032
+
1033
+ ```bash
1034
+ zenml feature-store register FEATURE_STORE_NAME --flavor FEATURE_STORE_FLAVOR [--FEATURE_STORE_OPTIONS]
1035
+ ```
1036
+
1037
+ You can also add any label to your stack component using the `--label` or `-l` flag:
1038
+
1039
+ ```bash
1040
+ zenml feature-store register FEATURE_STORE_NAME --flavor FEATURE_STORE_FLAVOR -l key1=value1 -l key2=value2
1041
+ ```
1042
+
1043
+ As you can see from the command above, when you register a new feature store,
1044
+ you have to choose a flavor. To see the full list of available feature store
1045
+ flavors, you can use the command:
1046
+
1047
+ ```bash
1048
+ zenml feature-store flavor list
1049
+ ```
1050
+
1051
+ This list will show you which integration these flavors belong to and which
1052
+ service connectors they are adaptable with. If you would like to get additional
1053
+ information regarding a specific flavor, you can utilize the command:
1054
+
1055
+ Note: Currently, ZenML only supports connecting to a Redis-backed Feast feature
1056
+ store as a stack component integration.
1057
+
1058
+ ```bash
1059
+ zenml feature-store flavor describe FLAVOR_NAME
1060
+ ```
1061
+
1062
+ To list all feature stores available and registered for use, use the
1063
+ `list` command:
1064
+
1065
+ ```bash
1066
+ zenml feature-store list
1067
+ ```
1068
+
1069
+ If you want the name of the feature store in the active stack, use the `get`
1070
+ command:
1071
+
1072
+ ```bash
1073
+ zenml feature-store get
1074
+ ```
1075
+
1076
+ For details about a particular feature store, use the `describe` command.
1077
+ By default, (without a specific feature store name passed in) it will describe
1078
+ the active or currently-used feature store:
1079
+
1080
+ ```bash
1081
+ zenml feature-store describe [FEATURE_STORE_NAME]
1082
+ ```
1083
+
1084
+ If you wish to update/rename a feature store, you can use the following commands
1085
+ respectively:
1086
+
1087
+ ```bash
1088
+ zenml feature-store update FEATURE_STORE_NAME --property_to_update=new_value
1089
+ zenml feature-store rename FEATURE_STORE_OLD_NAME FEATURE_STORE_NEW_NAME
1090
+ ```
1091
+
1092
+ To delete a feature store (and all of its contents), use the `delete`
1093
+ command:
1094
+
1095
+ ```bash
1096
+ zenml feature-store delete FEATURE_STORE_NAME
1097
+ ```
1098
+
1099
+ If you would like to connect/disconnect your feature store to/from a
1100
+ service connector, you can use the following commands:
1101
+
1102
+ ```bash
1103
+ zenml feature-store connect FEATURE_STORE_NAME -c CONNECTOR_NAME
1104
+ zenml feature-store disconnect
1105
+ ```
1106
+
1107
+ The ZenML CLI provides a few more utility functions for you to manage your
1108
+ feature stores. In order to get a full list of available functions,
1109
+ use the command:
1110
+
1111
+ ```bash
1112
+ zenml feature-store --help
1113
+ ```
1114
+
1115
+ Annotators
1116
+ ----------
1117
+
1118
+ [Annotators](https://docs.zenml.io/stack-components/annotators)
1119
+ enable the use of data annotation as part of your ZenML stack and pipelines.
1120
+
1121
+ By default, a default ZenML local stack will not register an annotator. If
1122
+ you wish to register a new annotator, do so with the `register` command:
1123
+
1124
+ ```bash
1125
+ zenml annotator register ANNOTATOR_NAME --flavor ANNOTATOR_FLAVOR [--ANNOTATOR_OPTIONS]
1126
+ ```
1127
+
1128
+ You can also add any label to your stack component using the `--label` or `-l` flag:
1129
+
1130
+ ```bash
1131
+ zenml annotator register ANNOTATOR_NAME --flavor ANNOTATOR_FLAVOR -l key1=value1 -l key2=value2
1132
+ ```
1133
+
1134
+ As you can see from the command above, when you register a new annotator,
1135
+ you have to choose a flavor. To see the full list of available annotator
1136
+ flavors, you can use the command:
1137
+
1138
+ ```bash
1139
+ zenml annotator flavor list
1140
+ ```
1141
+
1142
+ This list will show you which integration these flavors belong to and which
1143
+ service connectors they are adaptable with. If you would like to get additional
1144
+ information regarding a specific flavor, you can utilize the command:
1145
+
1146
+ ```bash
1147
+ zenml annotator flavor describe FLAVOR_NAME
1148
+ ```
1149
+
1150
+ To list all annotator available and registered for use, use the `list` command:
1151
+
1152
+ ```bash
1153
+ zenml annotator list
1154
+ ```
1155
+
1156
+ If you want the name of the annotator in the active stack, use the `get`
1157
+ command:
1158
+
1159
+ ```bash
1160
+ zenml annotator get
1161
+ ```
1162
+
1163
+ For details about a particular annotator, use the `describe` command.
1164
+ By default, (without a specific annotator name passed in) it will describe
1165
+ the active or currently used annotator:
1166
+
1167
+ ```bash
1168
+ zenml annotator describe [ANNOTATOR_NAME]
1169
+ ```
1170
+
1171
+ If you wish to update/rename an annotator, you can use the following commands
1172
+ respectively:
1173
+
1174
+ ```bash
1175
+ zenml annotator update ANNOTATOR_NAME --property_to_update=new_value
1176
+ zenml annotator rename ANNOTATOR_OLD_NAME ANNOTATOR_NEW_NAME
1177
+ ```
1178
+
1179
+ To delete an annotator (and all of its contents), use the `delete` command:
1180
+
1181
+ ```bash
1182
+ zenml annotator delete ANNOTATOR_NAME
1183
+ ```
1184
+
1185
+ If you would like to connect/disconnect your annotator to/from a service
1186
+ connector, you can use the following commands:
1187
+
1188
+ ```bash
1189
+ zenml annotator connect ANNOTATOR_NAME -c CONNECTOR_NAME
1190
+ zenml annotator disconnect
1191
+ ```
1192
+
1193
+ Finally, you can use the `dataset` command to interact with your annotation
1194
+ datasets:
1195
+
1196
+ ```bash
1197
+ zenml annotator dataset --help
1198
+ ```
1199
+
1200
+ The ZenML CLI provides a few more utility functions for you to manage your
1201
+ annotator. In order to get a full list of available functions, use the command:
1202
+
1203
+ ```bash
1204
+ zenml annotator --help
1205
+ ```
1206
+
1207
+ Image Builders
1208
+ --------------
1209
+
1210
+ In ZenML, [image builders](https://docs.zenml.io/stack-components/image-builders)
1211
+ allow you to build container images such
1212
+ that your machine-learning pipelines and steps can be executed in remote
1213
+ environments.
1214
+
1215
+ By default, a default ZenML local stack will not register an image builder. If
1216
+ you wish to register a new image builder, do so with the `register` command:
1217
+
1218
+ ```bash
1219
+ zenml image-builder register IMAGE_BUILDER_NAME --flavor IMAGE_BUILDER_FLAVOR [--IMAGE_BUILDER_OPTIONS]
1220
+ ```
1221
+
1222
+ You can also add any label to your stack component using the `--label` or `-l` flag:
1223
+
1224
+ ```bash
1225
+ zenml image-builder register IMAGE_BUILDER_NAME --flavor IMAGE_BUILDER_FLAVOR -l key1=value1 -l key2=value2
1226
+ ```
1227
+
1228
+ As you can see from the command above, when you register a new image builder,
1229
+ you have to choose a flavor. To see the full list of available image builder
1230
+ flavors, you can use the command:
1231
+
1232
+ ```bash
1233
+ zenml image-builder flavor list
1234
+ ```
1235
+
1236
+ This list will show you which integration these flavors belong to and which
1237
+ service connectors they are adaptable with. If you would like to get additional
1238
+ information regarding a specific flavor, you can utilize the command:
1239
+
1240
+ ```bash
1241
+ zenml image-builder flavor describe FLAVOR_NAME
1242
+ ```
1243
+
1244
+ To list all image builders available and registered for use, use the `list`
1245
+ command:
1246
+
1247
+ ```bash
1248
+ zenml image-builder list
1249
+ ```
1250
+
1251
+ If you want the name of the image builder in the active stack, use the `get`
1252
+ command:
1253
+
1254
+ ```bash
1255
+ zenml image-builder get
1256
+ ```
1257
+
1258
+ For details about a particular image builder, use the `describe` command.
1259
+ By default, (without a specific image builder name passed in) it will describe
1260
+ the active or currently used image builder:
1261
+
1262
+ ```bash
1263
+ zenml image-builder describe [IMAGE_BUILDER_NAME]
1264
+ ```
1265
+
1266
+ If you wish to update/rename an image builder, you can use the following
1267
+ commands respectively:
1268
+
1269
+ ```bash
1270
+ zenml image-builder update IMAGE_BUILDER_NAME --property_to_update=new_value
1271
+ zenml image-builder rename IMAGE_BUILDER_OLD_NAME IMAGE_BUILDER_NEW_NAME
1272
+ ```
1273
+
1274
+ To delete a image builder (and all of its contents), use the `delete` command:
1275
+
1276
+ ```bash
1277
+ zenml image-builder delete IMAGE_BUILDER_NAME
1278
+ ```
1279
+
1280
+ If you would like to connect/disconnect your image builder to/from a service
1281
+ connector, you can use the following commands:
1282
+
1283
+ ```bash
1284
+ zenml image-builder connect IMAGE_BUILDER_NAME -c CONNECTOR_NAME
1285
+ zenml image-builder disconnect
1286
+ ```
1287
+
1288
+ The ZenML CLI provides a few more utility functions for you to manage your
1289
+ image builders. In order to get a full list of available functions, use the
1290
+ command:
1291
+
1292
+ ```bash
1293
+ zenml image-builder --help
1294
+ ```
1295
+
1296
+ Model Registries
1297
+ ----------------
1298
+
1299
+ [Model registries](https://docs.zenml.io/stack-components/model-registries)
1300
+ are centralized repositories that facilitate the collaboration and management
1301
+ of machine learning models. To get a better understanding regarding model
1302
+ registries as a concept, use the command:
1303
+
1304
+ ```bash
1305
+ zenml model-registry explain
1306
+ ```
1307
+
1308
+ By default, a default ZenML local stack will not register a model registry. If
1309
+ you wish to register a new model registry, do so with the `register` command:
1310
+
1311
+ ```bash
1312
+ zenml model-registry register MODEL_REGISTRY_NAME --flavor MODEL_REGISTRY_FLAVOR [--MODEL_REGISTRY_OPTIONS]
1313
+ ```
1314
+
1315
+ You can also add any label to your stack component using the `--label` or `-l` flag:
1316
+
1317
+ ```bash
1318
+ zenml model-registry register MODEL_REGISTRY_NAME --flavor MODEL_REGISTRY_FLAVOR -l key1=value1 -l key2=value2
1319
+ ```
1320
+
1321
+ As you can see from the command above, when you register a new model registry,
1322
+ you have to choose a flavor. To see the full list of available model registry
1323
+ flavors, you can use the command:
1324
+
1325
+ ```bash
1326
+ zenml model-registry flavor list
1327
+ ```
1328
+
1329
+ This list will show you which integration these flavors belong to and which
1330
+ service connectors they are adaptable with. If you would like to get additional
1331
+ information regarding a specific flavor, you can utilize the command:
1332
+
1333
+ ```bash
1334
+ zenml model-registry flavor describe FLAVOR_NAME
1335
+ ```
1336
+
1337
+ To list all model registries available and registered for use, use the
1338
+ `list` command:
1339
+
1340
+ ```bash
1341
+ zenml model-registry list
1342
+ ```
1343
+
1344
+ If you want the name of the model registry in the active stack, use the `get`
1345
+ command:
1346
+
1347
+ ```bash
1348
+ zenml model-registry get
1349
+ ```
1350
+
1351
+ For details about a particular model registry, use the `describe` command.
1352
+ By default, (without a specific operator name passed in) it will describe the
1353
+ active or currently used model registry:
1354
+
1355
+ ```bash
1356
+ zenml model-registry describe [MODEL_REGISTRY_NAME]
1357
+ ```
1358
+
1359
+ If you wish to update/rename a model registry, you can use the following commands
1360
+ respectively:
1361
+
1362
+ ```bash
1363
+ zenml model-registry update MODEL_REGISTRY_NAME --property_to_update=new_value
1364
+ zenml model-registry rename MODEL_REGISTRY_OLD_NAME MODEL_REGISTRY_NEW_NAME
1365
+ ```
1366
+
1367
+ To delete a model registry (and all of its contents), use the `delete`
1368
+ command:
1369
+
1370
+ ```bash
1371
+ zenml model-registry delete MODEL_REGISTRY_NAME
1372
+ ```
1373
+
1374
+ If you would like to connect/disconnect your model registry to/from a
1375
+ service connector, you can use the following commands:
1376
+
1377
+ ```bash
1378
+ zenml model-registry connect MODEL_REGISTRY_NAME -c CONNECTOR_NAME
1379
+ zenml model-registry disconnect
1380
+ ```
1381
+
1382
+ The ZenML CLI provides a few more utility functions for you to manage your
1383
+ model registries. In order to get a full list of available functions,
1384
+ use the command:
1385
+
1386
+ ```bash
1387
+ zenml model-registry --help
1388
+ ```
1389
+
1390
+ Managing your Stacks
1391
+ --------------------
1392
+
1393
+ [The stack](https://docs.zenml.io/user-guide/production-guide/understand-stacks)
1394
+ is a grouping of your artifact store, your orchestrator, and other
1395
+ optional MLOps tools like experiment trackers or model deployers.
1396
+ With the ZenML tool, switching from a local stack to a distributed cloud
1397
+ environment can be accomplished with just a few CLI commands.
1398
+
1399
+ To register a new stack, you must already have registered the individual
1400
+ components of the stack using the commands listed above.
1401
+
1402
+ Use the ``zenml stack register`` command to register your stack. It
1403
+ takes four arguments as in the following example:
1404
+
1405
+ ```bash
1406
+ zenml stack register STACK_NAME \
1407
+ -a ARTIFACT_STORE_NAME \
1408
+ -o ORCHESTRATOR_NAME
631
1409
  ```
632
1410
 
633
1411
  Each corresponding argument should be the name, id or even the first few letters
634
1412
  of the id that uniquely identify the artifact store or orchestrator.
635
1413
 
1414
+ To create a new stack using the new service connector with a set of minimal components,
1415
+ use the following command:
1416
+
1417
+ ```bash
1418
+ zenml stack register STACK_NAME \
1419
+ -p CLOUD_PROVIDER
1420
+ ```
1421
+
1422
+ To create a new stack using the existing service connector with a set of minimal components,
1423
+ use the following command:
1424
+
1425
+ ```bash
1426
+ zenml stack register STACK_NAME \
1427
+ -sc SERVICE_CONNECTOR_NAME
1428
+ ```
1429
+
1430
+ To create a new stack using the existing service connector with existing components (
1431
+ important, that the components are already registered in the service connector), use the
1432
+ following command:
1433
+
1434
+ ```bash
1435
+ zenml stack register STACK_NAME \
1436
+ -sc SERVICE_CONNECTOR_NAME \
1437
+ -a ARTIFACT_STORE_NAME \
1438
+ -o ORCHESTRATOR_NAME \
1439
+ ...
1440
+ ```
1441
+
636
1442
  If you want to immediately set this newly created stack as your active stack,
637
1443
  simply pass along the `--set` flag.
638
1444
 
@@ -671,7 +1477,7 @@ If you want to copy a stack, run the following command:
671
1477
  zenml stack copy SOURCE_STACK_NAME TARGET_STACK_NAME
672
1478
  ```
673
1479
 
674
- If you wish to transfer one of your stacks to another machine, you can do so
1480
+ If you wish to transfer one of your stacks to another machine, you can do so
675
1481
  by exporting the stack configuration and then importing it again.
676
1482
 
677
1483
  To export a stack to YAML, run the following command:
@@ -719,6 +1525,13 @@ If you wish to rename your stack, use the following command:
719
1525
  zenml stack rename STACK_NAME NEW_STACK_NAME
720
1526
  ```
721
1527
 
1528
+ If you would like to export the requirements of your stack, you can
1529
+ use the command:
1530
+
1531
+ ```bash
1532
+ zenml stack export-requirements <STACK_NAME>
1533
+ ```
1534
+
722
1535
  If you want to copy a stack component, run the following command:
723
1536
  ```bash
724
1537
  zenml STACK_COMPONENT copy SOURCE_COMPONENT_NAME TARGET_COMPONENT_NAME
@@ -755,76 +1568,63 @@ following command:
755
1568
  zenml stack register-secrets [<STACK_NAME>]
756
1569
  ```
757
1570
 
758
- Administering your Code Repositories
759
- ------------------------------------
760
-
761
- Code repositories enable ZenML to keep track of the code version that you use
762
- for your pipeline runs. Additionally, running a pipeline which is tracked in
763
- a registered code repository can decrease the time it takes Docker to build images for
764
- containerized stack components.
1571
+ If you want to connect a service connector to a stack's components, you can use
1572
+ the `connect` command:
765
1573
 
766
- To register a code repository, use the following CLI
767
- command:
768
1574
  ```shell
769
- zenml code-repository register <NAME> --type=<CODE_REPOSITORY_TYPE] \
770
- [--CODE_REPOSITORY_OPTIONS]
1575
+ zenml stack connect STACK_NAME -c CONNECTOR_NAME
771
1576
  ```
772
1577
 
773
- ZenML currently supports code repositories of type `github` and `gitlab`, but
774
- you can also use your custom code repository implementation by passing the
775
- type `custom` and a source of your repository class.
776
-
777
- ```shell
778
- zenml code-repository register <NAME> --type=custom \
779
- --source=<CODE_REPOSITORY_SOURCE> [--CODE_REPOSITORY_OPTIONS]
780
- ```
781
-
782
- The `CODE_REPOSITORY_OPTIONS` depend on the configuration necessary for the
783
- type of code repository that you're using.
1578
+ Note that this only connects the service connector to the current components
1579
+ of the stack and not to the stack itself, which means that you need to rerun
1580
+ the command after adding new components to the stack.
784
1581
 
785
- If you want to list your registered code repositories, run:
786
- ```shell
787
- zenml code-repository list
788
- ```
1582
+ The ZenML CLI provides a few more utility functions for you to manage your
1583
+ stacks. In order to get a full list of available functions, use the command:
789
1584
 
790
- You can delete one of your registered code repositories like this:
791
- ```shell
792
- zenml code-repository delete <REPOSITORY_NAME_OR_ID>
1585
+ ```bash
1586
+ zenml stack --help
793
1587
  ```
794
1588
 
795
- Administering your Models
796
- ----------------------------
1589
+ Managing your Models
1590
+ --------------------
797
1591
 
798
1592
  ZenML provides several CLI commands to help you administer your models and
799
- their versions as part of the Model Control Plane.
1593
+ their versions as part of [the Model Control Plane](https://docs.zenml.io/user-guide/starter-guide/track-ml-models).
800
1594
 
801
1595
  To register a new model, you can use the following CLI command:
1596
+
802
1597
  ```bash
803
1598
  zenml model register --name <NAME> [--MODEL_OPTIONS]
804
1599
  ```
805
1600
 
806
1601
  To list all registered models, use:
1602
+
807
1603
  ```bash
808
1604
  zenml model list [MODEL_FILTER_OPTIONS]
809
1605
  ```
810
1606
 
811
1607
  To update a model, use:
1608
+
812
1609
  ```bash
813
1610
  zenml model update <MODEL_NAME_OR_ID> [--MODEL_OPTIONS]
814
1611
  ```
815
1612
 
816
1613
  If you would like to add or remove tags from the model, use:
1614
+
817
1615
  ```bash
818
- zenml model update <MODEL_NAME_OR_ID> --tag <TAG> --tag <TAG> ..
1616
+ zenml model update <MODEL_NAME_OR_ID> --tag <TAG> --tag <TAG> ..
819
1617
  --remove-tag <TAG> --remove-tag <TAG> ..
820
1618
  ```
821
1619
 
822
1620
  To delete a model, use:
1621
+
823
1622
  ```bash
824
1623
  zenml model delete <MODEL_NAME_OR_ID>
825
1624
  ```
826
1625
 
827
1626
  The CLI interface for models also helps to navigate through artifacts linked to a specific model versions.
1627
+
828
1628
  ```bash
829
1629
  zenml model data_artifacts <MODEL_NAME_OR_ID> [-v <VERSION>]
830
1630
  zenml model deployment_artifacts <MODEL_NAME_OR_ID> [-v <VERSION>]
@@ -832,40 +1632,49 @@ zenml model model_artifacts <MODEL_NAME_OR_ID> [-v <VERSION>]
832
1632
  ```
833
1633
 
834
1634
  You can also navigate the pipeline runs linked to a specific model versions:
1635
+
835
1636
  ```bash
836
1637
  zenml model runs <MODEL_NAME_OR_ID> [-v <VERSION>]
837
1638
  ```
838
1639
 
839
1640
  To list the model versions of a specific model, use:
1641
+
840
1642
  ```bash
841
1643
  zenml model version list [--model-name <MODEL_NAME> --name <MODEL_VERSION_NAME> OTHER_OPTIONS]
842
1644
  ```
843
1645
 
844
1646
  To delete a model version, use:
1647
+
845
1648
  ```bash
846
1649
  zenml model version delete <MODEL_NAME_OR_ID> <VERSION>
847
1650
  ```
848
1651
 
849
1652
  To update a model version, use:
1653
+
850
1654
  ```bash
851
1655
  zenml model version update <MODEL_NAME_OR_ID> <VERSION> [--MODEL_VERSION_OPTIONS]
852
1656
  ```
1657
+
853
1658
  These are some of the more common uses of model version updates:
1659
+
854
1660
  - stage (i.e. promotion)
1661
+
855
1662
  ```bash
856
1663
  zenml model version update <MODEL_NAME_OR_ID> <VERSION> --stage <STAGE>
857
1664
  ```
1665
+
858
1666
  - tags
1667
+
859
1668
  ```bash
860
- zenml model version update <MODEL_NAME_OR_ID> <VERSION> --tag <TAG> --tag <TAG> ..
1669
+ zenml model version update <MODEL_NAME_OR_ID> <VERSION> --tag <TAG> --tag <TAG> ..
861
1670
  --remove-tag <TAG> --remove-tag <TAG> ..
862
1671
  ```
863
1672
 
864
- Administering your Pipelines
865
- ----------------------------
1673
+ Managing your Pipelines & Artifacts
1674
+ -----------------------------------
866
1675
 
867
- ZenML provides several CLI commands to help you administer your pipelines and
868
- pipeline runs.
1676
+ ZenML provides several CLI commands to help you [administer your pipelines and
1677
+ pipeline runs](https://docs.zenml.io/user-guide/starter-guide/manage-artifacts).
869
1678
 
870
1679
  To explicitly register a pipeline you need to point to a pipeline instance
871
1680
  in your Python code. Let's say you have a Python file called `run.py` and
@@ -882,7 +1691,7 @@ def my_pipeline(...):
882
1691
 
883
1692
  You can register your pipeline like this:
884
1693
  ```bash
885
- zenml pipeline register my_pipeline
1694
+ zenml pipeline register run.my_pipeline
886
1695
  ```
887
1696
 
888
1697
  To list all registered pipelines, use:
@@ -891,15 +1700,15 @@ To list all registered pipelines, use:
891
1700
  zenml pipeline list
892
1701
  ```
893
1702
 
894
- Since every pipeline run creates a new pipeline by default, you might
895
- occasionally want to delete a pipeline, which you can do via:
1703
+ To delete a pipeline, run:
896
1704
 
897
1705
  ```bash
898
1706
  zenml pipeline delete <PIPELINE_NAME>
899
1707
  ```
900
1708
 
901
- This will delete the pipeline and change all corresponding pipeline runs to
902
- become unlisted (not linked to any pipeline).
1709
+ This will delete the pipeline and change all corresponding
1710
+ pipeline runs to become unlisted (not linked to any pipeline).
1711
+
903
1712
 
904
1713
  To list all pipeline runs that you have executed, use:
905
1714
 
@@ -913,6 +1722,13 @@ To delete a pipeline run, use:
913
1722
  zenml pipeline runs delete <PIPELINE_RUN_NAME_OR_ID>
914
1723
  ```
915
1724
 
1725
+ To refresh the status of a pipeline run, you can use the `refresh` command (
1726
+ only supported for pipelines executed on Vertex, Sagemaker or AzureML).
1727
+
1728
+ ```bash
1729
+ zenml pipeline runs refresh <PIPELINE_RUN_NAME_OR_ID>
1730
+ ```
1731
+
916
1732
  If you run any of your pipelines with `pipeline.run(schedule=...)`, ZenML keeps
917
1733
  track of the schedule and you can list all schedules via:
918
1734
 
@@ -928,7 +1744,7 @@ zenml pipeline schedule delete <SCHEDULE_NAME_OR_ID>
928
1744
 
929
1745
  Note, however, that this will only delete the reference saved in ZenML and does
930
1746
  NOT stop/delete the schedule in the respective orchestrator. This still needs to
931
- be done manually. For example, using the Airflow orchestrator you would have
1747
+ be done manually. For example, using the Airflow orchestrator you would have
932
1748
  to open the web UI to manually click to stop the schedule from executing.
933
1749
 
934
1750
  Each pipeline run automatically saves its artifacts in the artifact store. To
@@ -953,19 +1769,25 @@ zenml artifact update <NAME> -t <TAG1> -t <TAG2> -r <TAG_TO_REMOVE>
953
1769
  zenml artifact version update <NAME> -v <VERSION> -t <TAG1> -t <TAG2> -r <TAG_TO_REMOVE>
954
1770
  ```
955
1771
 
956
- The metadata of artifacts or artifact versions stored by ZenML can only be
1772
+ The metadata of artifacts or artifact versions stored by ZenML can only be
957
1773
  deleted once they are no longer used by any pipeline runs. I.e., an artifact
958
1774
  version can only be deleted if the run that produced it and all runs that used
959
1775
  it as an input have been deleted. Similarly, an artifact can only be deleted if
960
1776
  all its versions can be deleted.
961
1777
 
962
- To delete all artifacts and artifact versions that are no longer linked to any
1778
+ To delete all artifacts and artifact versions that are no longer linked to any
963
1779
  pipeline runs, use:
964
1780
 
965
1781
  ```bash
966
1782
  zenml artifact prune
967
1783
  ```
968
1784
 
1785
+ You might find that some artifacts throw errors when you try to prune them,
1786
+ likely because they were stored locally and no longer exist. If you wish to
1787
+ continue pruning and to ignore these errors, please add the `--ignore-errors`
1788
+ flag. Warning messages will still be output to the terminal during this
1789
+ process.
1790
+
969
1791
  Each pipeline run that requires Docker images also stores a build which
970
1792
  contains the image names used for this run. To list all builds, use:
971
1793
 
@@ -979,308 +1801,419 @@ To delete a specific build, use:
979
1801
  zenml pipeline builds delete <BUILD_ID>
980
1802
  ```
981
1803
 
982
- Building an image without running your Pipelines
1804
+ Managing the local ZenML Dashboard
983
1805
  ----------------------------------
984
1806
 
985
- To build Docker images for your pipeline without actually running the pipeline,
986
- use:
1807
+ The ZenML dashboard is a web-based UI that allows you to visualize and navigate
1808
+ the stack configurations, pipelines and pipeline runs tracked by ZenML among
1809
+ other things. You can start the ZenML dashboard locally by running the following
1810
+ command:
987
1811
 
988
1812
  ```bash
989
- zenml pipeline build <PIPELINE_ID_OR_NAME>
1813
+ zenml login --local
990
1814
  ```
991
1815
 
992
- To specify settings for the Docker builds, use the `--config/-c` option of the
993
- command. For more information about the structure of this configuration file,
994
- check out the `zenml.pipelines.base_pipeline.BasePipeline.build(...)` method.
1816
+ This will start the dashboard on your local machine where you can access it at
1817
+ the URL printed to the console.
1818
+
1819
+ If you have closed the dashboard in your browser and want to open it again,
1820
+ you can run:
1821
+
1822
+ ```bash
1823
+ zenml show
1824
+ ```
1825
+
1826
+ If you want to stop the dashboard, simply run:
995
1827
 
996
1828
  ```bash
997
- zenml pipeline build <PIPELINE_ID_OR_NAME> --config=<PATH_TO_CONFIG_YAML>
1829
+ zenml logout --local
998
1830
  ```
999
1831
 
1000
- If you want to build the pipeline for a stack different than your current active
1001
- stack, use the `--stack` option.
1832
+ The `zenml login --local` command has a few additional options that you can use
1833
+ to customize how the ZenML dashboard is running.
1834
+
1835
+ By default, the dashboard is started as a background process. On some operating
1836
+ systems, this capability is not available. In this case, you can use the
1837
+ `--blocking` flag to start the dashboard in the foreground:
1838
+
1002
1839
  ```bash
1003
- zenml pipeline build <PIPELINE_ID_OR_NAME> --stack=<STACK_ID_OR_NAME>
1840
+ zenml login --local --blocking
1004
1841
  ```
1005
1842
 
1843
+ This will block the terminal until you stop the dashboard with CTRL-C.
1006
1844
 
1007
- To run a pipeline that was previously registered, use:
1845
+ Another option you can use, if you have Docker installed on your machine, is to
1846
+ run the dashboard in a Docker container. This is useful if you don't want to
1847
+ install all the Zenml server dependencies on your machine. To do so, simply run:
1848
+
1849
+ ```bash
1850
+ zenml login --local --docker
1851
+ ```
1852
+
1853
+ The TCP port and the host address that the dashboard uses to listen for
1854
+ connections can also be customized. Using an IP address that is not the default
1855
+ `localhost` or 127.0.0.1 is especially useful if you're running some type of
1856
+ local ZenML orchestrator, such as the k3d Kubeflow orchestrator or Docker
1857
+ orchestrator, that cannot directly connect to the local ZenML server.
1858
+
1859
+ For example, to start the dashboard on port 9000 and have it listen
1860
+ on all locally available interfaces on your machine, run:
1861
+
1862
+ ```bash
1863
+ zenml login --local --port 9000 --ip-address 0.0.0.0
1864
+ ```
1865
+
1866
+ Note that the above 0.0.0.0 IP address also exposes your ZenML dashboard
1867
+ externally through your public interface. Alternatively, you can choose an
1868
+ explicit IP address that is configured on one of your local interfaces, such as
1869
+ the Docker bridge interface, which usually has the IP address `172.17.0.1`:
1870
+
1871
+ ```bash
1872
+ zenml login --local --port 9000 --ip-address 172.17.0.1
1873
+ ```
1874
+
1875
+ If you would like to take a look at the logs for the local ZenML server:
1876
+
1877
+ ```bash
1878
+ zenml logs
1879
+ ```
1880
+
1881
+ Connecting to a ZenML Server
1882
+ ----------------------------
1883
+
1884
+ The ZenML client can be [configured to connect to a local ZenML server, a remote
1885
+ database or a remote ZenML server](https://docs.zenml.io/how-to/connecting-to-zenml)
1886
+ with the `zenml login` command.
1887
+
1888
+ To connect or re-connect to any ZenML server, if you know its URL, you can
1889
+ simply run:
1890
+
1891
+ ```bash
1892
+ zenml login https://zenml.example.com:8080
1893
+ ```
1894
+
1895
+ Running `zenml login` without any arguments will check if your current ZenML
1896
+ server session is still valid. If it is not, you will be prompted to log in
1897
+ again. This is useful if you quickly want to refresh your CLI session when
1898
+ the current session expires.
1899
+
1900
+ You can open the ZenML dashboard of your currently connected ZenML server using
1901
+ the following command:
1902
+
1903
+ ```bash
1904
+ zenml server show
1905
+ ```
1906
+
1907
+ Note that if you have set your `AUTO_OPEN_DASHBOARD` environment variable to
1908
+ `false` then this will not open the dashboard until you set it back to `true`.
1909
+
1910
+ The CLI can be authenticated to multiple ZenML servers at the same time,
1911
+ even though it can only be connected to one server at a time. You can list
1912
+ all the ZenML servers that the client is currently authenticated to by
1913
+ running:
1008
1914
 
1009
1915
  ```bash
1010
- zenml pipeline run <PIPELINE_ID_OR_NAME>
1916
+ zenml server list
1011
1917
  ```
1012
1918
 
1013
- To specify settings for the pipeline, use the `--config/-c` option of the
1014
- command. For more information about the structure of this configuration file,
1015
- check out the `zenml.pipelines.base_pipeline.BasePipeline.run(...)` method.
1919
+ To disconnect from the current ZenML server and revert to using the local
1920
+ default database, use the following command:
1016
1921
 
1017
1922
  ```bash
1018
- zenml pipeline run <PIPELINE_ID_OR_NAME> --config=<PATH_TO_CONFIG_YAML>
1923
+ zenml logout
1019
1924
  ```
1020
1925
 
1021
- If you want to run the pipeline on a stack different than your current active
1022
- stack, use the `--stack` option.
1926
+ You can inspect the current ZenML configuration at any given time using the
1927
+ following command:
1928
+
1023
1929
  ```bash
1024
- zenml pipeline run <PIPELINE_ID_OR_NAME> --stack=<STACK_ID_OR_NAME>
1930
+ zenml status
1025
1931
  ```
1026
1932
 
1027
- Managing the local ZenML Dashboard
1028
- ----------------------------------
1933
+ Example output:
1029
1934
 
1030
- The ZenML dashboard is a web-based UI that allows you to visualize and navigate
1031
- the stack configurations, pipelines and pipeline runs tracked by ZenML among
1032
- other things. You can start the ZenML dashboard locally by running the following
1033
- command:
1935
+ ```
1936
+ $ zenml status
1937
+ -----ZenML Client Status-----
1938
+ Connected to a ZenML Pro server: `test-zenml-login` [16f8a35d-5c2f-44aa-a564-b34186fbf6d6]
1939
+ ZenML Pro Organization: My Organization
1940
+ ZenML Pro authentication: valid until 2024-10-26 10:18:51 CEST (in 20h37m9s)
1941
+ Dashboard: https://cloud.zenml.io/organizations/bf873af9-aaf9-4ad1-a08e-3dc6d920d590/tenants/16f8336d-5c2f-44aa-a534-b34186fbf6d6
1942
+ API: https://6784e58f-zenml.staging.cloudinfra.zenml.io
1943
+ Server status: 'available'
1944
+ Server authentication: never expires
1945
+ The active user is: 'user'
1946
+ The active stack is: 'default' (global)
1947
+ Using configuration from: '/home/user/.config/zenml'
1948
+ Local store files are located at: '/home/user/.config/zenml/local_stores'
1034
1949
 
1035
- ```bash
1036
- zenml up
1950
+ -----Local ZenML Server Status-----
1951
+ The local daemon server is running at: http://127.0.0.1:8237
1037
1952
  ```
1038
1953
 
1039
- This will start the dashboard on your local machine where you can access it at
1040
- the URL printed to the console.
1954
+ When connecting to a ZenML server using the web login flow, you will be provided
1955
+ with the option to `Trust this device`. If you opt out of it a 24-hour token
1956
+ will be issued for the authentication service. If you opt-in, you will be
1957
+ issued a 30-day token instead.
1041
1958
 
1042
- If you have closed the dashboard in your browser and want to open it again,
1043
- you can run:
1959
+ If you would like to see a list of all trusted devices, you can use:
1044
1960
 
1045
1961
  ```bash
1046
- zenml show
1962
+ zenml authorized-device list
1047
1963
  ```
1048
1964
 
1049
- If you want to stop the dashboard, simply run:
1965
+ or if you would like to get the details regarding a specific device,
1966
+ you can use:
1050
1967
 
1051
1968
  ```bash
1052
- zenml down
1969
+ zenml authorized-device describe DEVICE_ID_OR_PREFIX
1053
1970
  ```
1054
1971
 
1055
- The `zenml up` command has a few additional options that you can use to
1056
- customize how the ZenML dashboard is running.
1057
-
1058
- By default, the dashboard is started as a background process. On some operating
1059
- systems, this capability is not available. In this case, you can use the
1060
- `--blocking` flag to start the dashboard in the foreground:
1972
+ Alternatively, you can lock and unlock an authorized device by using the
1973
+ following commands:
1061
1974
 
1062
1975
  ```bash
1063
- zenml up --blocking
1976
+ zenml authorized-device lock DEVICE_ID_OR_PREFIX
1977
+ zenml authorized-device unlock DEVICE_ID_OR_PREFIX
1064
1978
  ```
1065
1979
 
1066
- This will block the terminal until you stop the dashboard with CTRL-C.
1067
-
1068
- Another option you can use, if you have Docker installed on your machine, is to
1069
- run the dashboard in a Docker container. This is useful if you don't want to
1070
- install all the Zenml server dependencies on your machine. To do so, simply run:
1980
+ Finally, you can remove an authorized device by using the `delete` command:
1071
1981
 
1072
1982
  ```bash
1073
- zenml up --docker
1983
+ zenml authorized-device delete DEVICE_ID_OR_PREFIX
1074
1984
  ```
1075
1985
 
1076
- The TCP port and the host address that the dashboard uses to listen for
1077
- connections can also be customized. Using an IP address that is not the default
1078
- `localhost` or 127.0.0.1 is especially useful if you're running some type of
1079
- local ZenML orchestrator, such as the k3d Kubeflow orchestrator or Docker
1080
- orchestrator, that cannot directly connect to the local ZenML server.
1986
+ Secrets management
1987
+ ------------------
1081
1988
 
1082
- For example, to start the dashboard on port 9000 and have it listen
1083
- on all locally available interfaces on your machine, run:
1989
+ ZenML offers a way to [securely store secrets associated with your other
1990
+ stack components and infrastructure](https://docs.zenml.io/getting-started/deploying-zenml/secret-management).
1991
+ A ZenML Secret is a collection or grouping of key-value pairs stored by the
1992
+ ZenML secrets store. ZenML Secrets are identified by a unique name which
1993
+ allows you to fetch or reference them in your pipelines and stacks.
1084
1994
 
1085
- ```bash
1086
- zenml up --port 9000 --ip-address 0.0.0.0
1087
- ```
1995
+ Depending on how you set up and deployed ZenML, the secrets store keeps secrets
1996
+ in the local database or uses the ZenML server your client is connected to:
1088
1997
 
1089
- Note that the above 0.0.0.0 IP address also exposes your ZenML dashboard
1090
- externally through your public interface. Alternatively, you can choose an
1091
- explicit IP address that is configured on one of your local interfaces, such as
1092
- the Docker bridge interface, which usually has the IP address `172.17.0.1`:
1998
+ * if you are using the default ZenML client settings, or if you connect your
1999
+ ZenML client to a local ZenML server started with `zenml login --local`, the
2000
+ secrets store is using the same local SQLite database as the rest of ZenML
2001
+ * if you connect your ZenML client to a remote ZenML server, the
2002
+ secrets are no longer managed on your local machine, but through the remote
2003
+ server instead. Secrets are stored in whatever secrets store back-end the
2004
+ remote server is configured to use. This can be a SQL database, one of the
2005
+ managed cloud secrets management services, or even a custom back-end.
2006
+
2007
+ To create a secret, use the `create` command and pass the key-value pairs
2008
+ as command-line arguments:
1093
2009
 
1094
2010
  ```bash
1095
- zenml up --port 9000 --ip-address 172.17.0.1
2011
+ zenml secret create SECRET_NAME --key1=value1 --key2=value2 --key3=value3 ...
2012
+
2013
+ # Another option is to use the '--values' option and provide key-value pairs in either JSON or YAML format.
2014
+ zenml secret create SECRET_NAME --values='{"key1":"value2","key2":"value2","key3":"value3"}'
1096
2015
  ```
1097
2016
 
1098
- Managing the global configuration
1099
- ---------------------------------
2017
+ Note that when using the previous command the keys and values will be preserved in your `bash_history` file, so
2018
+ you may prefer to use the interactive `create` command instead:
1100
2019
 
1101
- The ZenML global configuration CLI commands cover options such as enabling or
1102
- disabling the collection of anonymous usage statistics, changing the logging
1103
- verbosity and configuring your ZenML client to connect to a remote database or
1104
- ZenML server.
2020
+ ```shell
2021
+ zenml secret create SECRET_NAME -i
2022
+ ```
1105
2023
 
1106
- In order to help us better understand how the community uses ZenML, the library
1107
- reports anonymized usage statistics. You can always opt-out by using the CLI
1108
- command:
2024
+ As an alternative to the interactive mode, also useful for values that
2025
+ are long or contain newline or special characters, you can also use the special
2026
+ `@` syntax to indicate to ZenML that the value needs to be read from a file:
1109
2027
 
1110
2028
  ```bash
1111
- zenml analytics opt-out
2029
+ zenml secret create SECRET_NAME \
2030
+ --aws_access_key_id=1234567890 \
2031
+ --aws_secret_access_key=abcdefghij \
2032
+ --aws_session_token=@/path/to/token.txt
2033
+
2034
+ # Alternatively for providing key-value pairs, you can utilize the '--values' option by specifying a file path containing
2035
+ # key-value pairs in either JSON or YAML format.
2036
+ zenml secret create SECRET_NAME --values=@/path/to/token.txt
1112
2037
  ```
1113
2038
 
1114
- If you want to opt back in, use the following command:
2039
+ To list all the secrets available, use the `list` command:
1115
2040
 
1116
2041
  ```bash
1117
- zenml analytics opt-in
2042
+ zenml secret list
1118
2043
  ```
1119
2044
 
1120
- The verbosity of the ZenML client output can be configured using the
1121
- ``zenml logging`` command. For example, to set the verbosity to DEBUG, run:
2045
+ To get the key-value pairs for a particular secret, use the `get` command:
1122
2046
 
1123
2047
  ```bash
1124
- zenml logging set-verbosity DEBUG
2048
+ zenml secret get SECRET_NAME
1125
2049
  ```
1126
2050
 
1127
- The ZenML client can be configured to connect to a remote database or ZenML
1128
- server with the `zenml connect` command. If no arguments are supplied, ZenML
1129
- will attempt to connect to the last ZenML server deployed from the local host
1130
- using the 'zenml deploy' command:
2051
+ To update a secret, use the `update` command:
1131
2052
 
1132
2053
  ```bash
1133
- zenml connect
2054
+ zenml secret update SECRET_NAME --key1=value1 --key2=value2 --key3=value3 ...
2055
+
2056
+ # Another option is to use the '--values' option and provide key-value pairs in either JSON or YAML format.
2057
+ zenml secret update SECRET_NAME --values='{"key1":"value2","key2":"value2","key3":"value3"}'
1134
2058
  ```
1135
2059
 
1136
- To connect to a ZenML server, you can either pass the configuration as command
1137
- line arguments or as a YAML file:
2060
+ Note that when using the previous command the keys and values will be preserved in your `bash_history` file, so
2061
+ you may prefer to use the interactive `update` command instead:
1138
2062
 
1139
- ```bash
1140
- zenml connect --url=https://zenml.example.com:8080 --no-verify-ssl
2063
+ ```shell
2064
+ zenml secret update SECRET_NAME -i
1141
2065
  ```
1142
2066
 
1143
- or
2067
+ Finally, to delete a secret, use the `delete` command:
1144
2068
 
1145
2069
  ```bash
1146
- zenml connect --config=/path/to/zenml_server_config.yaml
2070
+ zenml secret delete SECRET_NAME
1147
2071
  ```
1148
2072
 
1149
- The YAML file should have the following structure when connecting to a ZenML
1150
- server:
2073
+ Secrets can be scoped to a workspace or a user. By default, secrets
2074
+ are scoped to the current workspace. To scope a secret to a user, use the
2075
+ `--scope user` argument in the `register` command.
2076
+
2077
+ Auth management
2078
+ ---------------
1151
2079
 
1152
- ```yaml
1153
- url: <The URL of the ZenML server>
1154
- username: <The username to use for authentication>
1155
- password: <The password to use for authentication>
1156
- verify_ssl: |
1157
- <Either a boolean, in which case it controls whether the
1158
- server's TLS certificate is verified, or a string, in which case it
1159
- must be a path to a CA certificate bundle to use or the CA bundle
1160
- value itself>
1161
- ```
2080
+ Building and maintaining an MLOps workflow can involve numerous third-party
2081
+ libraries and external services. In most cases, this ultimately presents a
2082
+ challenge in configuring uninterrupted, secure access to infrastructure
2083
+ resources. In ZenML, Service Connectors streamline this process by abstracting
2084
+ away the complexity of authentication and help you connect your stack to your
2085
+ resources. You can find the full docs on the ZenML service connectors
2086
+ [here](https://docs.zenml.io/how-to/infrastructure-deployment/auth-management).
2087
+
2088
+ The ZenML CLI features a variety of commands to help you manage your service
2089
+ connectors. First of all, to explore all the types of service connectors
2090
+ available in ZenML, you can use the following commands:
1162
2091
 
1163
- Example of a ZenML server YAML configuration file:
2092
+ ```bash
2093
+ # To get the complete list
2094
+ zenml service-connector list-types
1164
2095
 
1165
- ```yaml
1166
- url: https://ac8ef63af203226194a7725ee71d85a-7635928635.us-east-1.elb.amazonaws.com/zenml
1167
- username: admin
1168
- password: Pa$$word123
1169
- verify_ssl: |
1170
- -----BEGIN CERTIFICATE-----
1171
- MIIDETCCAfmgAwIBAgIQYUmQg2LR/pHAMZb/vQwwXjANBgkqhkiG9w0BAQsFADAT
1172
- MREwDwYDVQQDEwh6ZW5tbC1jYTAeFw0yMjA5MjYxMzI3NDhaFw0yMzA5MjYxMzI3
1173
- ...
1174
- ULnzA0JkRWRnFqH6uXeJo1KAVqtxn1xf8PYxx3NlNDr9wi8KKwARf2lwm6sH4mvq
1175
- 1aZ/0iYnGKCu7rLJzxeguliMf69E
1176
- -----END CERTIFICATE-----
2096
+ # To get the details regarding a single type
2097
+ zenml service-connector describe-type
1177
2098
  ```
1178
2099
 
1179
- Both options can be combined, in which case the command line arguments will
1180
- override the values in the YAML file. For example, it is possible and
1181
- recommended that you supply the password only as a command line argument:
2100
+ For each type of service connector, you will also see a list of supported
2101
+ resource types. These types provide a way for organizing different resources
2102
+ into logical classes based on the standard and/or protocol used to access them.
2103
+ In addition to the resource types, each type will feature a different set of
2104
+ authentication methods.
2105
+
2106
+ Once you decided which service connector to use, you can create it with the
2107
+ `register` command as follows:
1182
2108
 
1183
2109
  ```bash
1184
- zenml connect --username zenml --password=Pa@#$#word --config=/path/to/zenml_server_config.yaml
2110
+ zenml service-connector register SERVICE_CONNECTOR_NAME \
2111
+ --type TYPE [--description DESCRIPTION] [--resource-type RESOURCE_TYPE] \
2112
+ [--auth-method AUTH_METHOD] ...
1185
2113
  ```
1186
2114
 
1187
- You can open the ZenML dashboard of your currently connected ZenML server using
1188
- the following command:
2115
+ For more details on how to create a service connector, please refer to our
2116
+ [docs](https://docs.zenml.io/how-to/infrastructure-deployment/auth-management).
2117
+
2118
+ To check if your service connector is registered properly, you can `verify` it.
2119
+ By doing this, you can both check if it is configured correctly and also, you
2120
+ can fetch the list of resources it has access to:
1189
2121
 
1190
2122
  ```bash
1191
- zenml show
2123
+ zenml service-connector verify SERVICE_CONNECTOR_NAME_ID_OR_PREFIX
2124
+ ```
1192
2125
 
1193
- Note that if you have set your `AUTO_OPEN_DASHBOARD` environment variable to `false` then this will not open the dashboard until you set it back to `true`.
1194
- To disconnect from the current ZenML server and revert to using the local
1195
- default database, use the following command:
2126
+ Some service connectors come equipped with the capability of configuring
2127
+ the clients and SDKs on your local machine with the credentials inferred from
2128
+ your service connector. To use this functionality, simply use the `login`
2129
+ command:
1196
2130
 
1197
2131
  ```bash
1198
- zenml disconnect
2132
+ zenml service-connector login SERVICE_CONNECTOR_NAME_ID_OR_PREFIX
1199
2133
  ```
1200
2134
 
1201
- You can inspect the current ZenML configuration at any given time using the
1202
- following command:
2135
+ To list all the service connectors that you have registered, you can use:
1203
2136
 
1204
2137
  ```bash
1205
- zenml status
2138
+ zenml service-connector list
1206
2139
  ```
1207
2140
 
1208
- Example output:
2141
+ Moreover, if you would like to list all the resources accessible by your
2142
+ service connectors, you can use the following command:
1209
2143
 
1210
- ```
1211
- zenml status
1212
- Running without an active repository root.
1213
- Connected to a ZenML server: 'https://ac8ef63af203226194a7725ee71d85a-7635928635.us-east-1.elb.amazonaws.com'
1214
- The current user is: 'default'
1215
- The active workspace is: 'default' (global)
1216
- The active stack is: 'default' (global)
1217
- The status of the local dashboard:
1218
- ZenML server 'local'
1219
- ┏━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
1220
- ┃ URL │ http://172.17.0.1:9000 ┃
1221
- ┠────────────────┼─────────────────────────────┨
1222
- ┃ STATUS │ ✅ ┃
1223
- ┠────────────────┼─────────────────────────────┨
1224
- ┃ STATUS_MESSAGE │ Docker container is running ┃
1225
- ┠────────────────┼─────────────────────────────┨
1226
- ┃ CONNECTED │ ┃
1227
- ┗━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
2144
+ ```bash
2145
+ zenml service-connector list-resources [--resource-type RESOURCE_TYPE] /
2146
+ [--connector-type CONNECTOR_TYPE] ...
1228
2147
  ```
1229
2148
 
1230
- The ``zenml connect`` command can also be used to configure your client with
1231
- more advanced options, such as connecting directly to a local or remote SQL
1232
- database. In this case, the `--raw-config` flag must be passed to instruct the
1233
- CLI to not validate or fill in the missing configuration fields. For example,
1234
- to connect to a remote MySQL database, run:
2149
+ This command can possibly take a long time depending on the number of service
2150
+ connectors you have registered. Consider using the right filters when you are
2151
+ listing resources.
2152
+
2153
+ If you want to see the details about a specific service connector that you have
2154
+ registered, you can use the `describe` command:
1235
2155
 
1236
2156
  ```bash
1237
- zenml connect --raw-config --config=/path/to/mysql_config.yaml
2157
+ zenml service-connector describe SERVICE_CONNECTOR_NAME_ID_OR_PREFIX
1238
2158
  ```
1239
2159
 
1240
- with a YAML configuration file that looks like this:
2160
+ You can update a registered service connector by using the `update` command.
2161
+ Keep in mind that all service connector updates are validated before being
2162
+ applied. If you want to disable this behavior please use the `--no-verify`
2163
+ flag.
2164
+
2165
+ ```bash
2166
+ zenml service-connector update SERVICE_CONNECTOR_NAME_ID_OR_PREFIX ...
2167
+ ```
1241
2168
 
1242
- ```yaml
1243
- type: sql
1244
- url: mysql://<username>:<password>@mysql.database.com/<database_name>
1245
- ssl_ca: |
1246
- -----BEGIN CERTIFICATE-----
1247
- MIIEBjCCAu6gAwIBAgIJAMc0ZzaSUK51MA0GCSqGSIb3DQEBCwUAMIGPMQswCQYD
1248
- VQQGEwJVUzEQMA4GA1UEBwwHU2VhdHRsZTETMBEGA1UECAwKV2FzaGluZ3RvbjEi
1249
- MCAGA1UECgwZQW1hem9uIFdlYiBTZXJ2aWNlcywgSW5jLjETMBEGA1UECwwKQW1h
1250
- ...
1251
- KoZIzj0EAwMDaAAwZQIxAIqqZWCSrIkZ7zsv/FygtAusW6yvlL935YAWYPVXU30m
1252
- jkMFLM+/RJ9GMvnO8jHfCgIwB+whlkcItzE9CRQ6CsMo/d5cEHDUu/QW6jSIh9BR
1253
- OGh9pTYPVkUbBiKPA7lVVhre
1254
- -----END CERTIFICATE-----
2169
+ Finally, if you wish to remove a service connector, you can use the `delete`
2170
+ command:
1255
2171
 
1256
- ssl_cert: null
1257
- ssl_key: null
1258
- ssl_verify_server_cert: false
2172
+ ```bash
2173
+ zenml service-connector delete SERVICE_CONNECTOR_NAME_ID_OR_PREFIX
1259
2174
  ```
1260
2175
 
1261
- Managing users and workspaces
1262
- -------------------------------------------
2176
+ Managing users
2177
+ --------------
1263
2178
 
1264
- When using the ZenML service, you can manage permissions by managing users and
1265
- workspaces and using the CLI.
1266
- If you want to create a new user or delete an existing one, run either
2179
+ When using the ZenML service, you can manage permissions by managing users
2180
+ using the CLI. If you want to create a new user or delete an existing one,
2181
+ run either
1267
2182
 
1268
2183
  ```bash
1269
2184
  zenml user create USER_NAME
1270
- ```
1271
- or
1272
- ```bash
1273
2185
  zenml user delete USER_NAME
1274
2186
  ```
1275
2187
 
1276
2188
  To see a list of all users, run:
2189
+
1277
2190
  ```bash
1278
2191
  zenml user list
1279
2192
  ```
1280
2193
 
2194
+ For detail about the particular user, use the `describe` command. By default,
2195
+ (without a specific user name passed in) it will describe the active user:
2196
+
2197
+ ```bash
2198
+ zenml user describe [USER_NAME]
2199
+ ```
2200
+
2201
+ If you want to update any properties of a specific user, you can use the
2202
+ `update` command. Use the `--help` flag to get a full list of available
2203
+ properties to update:
2204
+
2205
+ ```bash
2206
+ zenml user update --help
2207
+ ```
2208
+
2209
+ If you want to change the password of the current user account:
2210
+
2211
+ ```bash
2212
+ zenml user change-password --help
2213
+ ```
1281
2214
 
1282
- Managing service accounts
1283
- -------------------------
2215
+ Service Accounts
2216
+ ----------------
1284
2217
 
1285
2218
  ZenML supports the use of service accounts to authenticate clients to the
1286
2219
  ZenML server using API keys. This is useful for automating tasks such as
@@ -1298,17 +2231,20 @@ then use the issued API key to connect your ZenML client to the server with the
1298
2231
  CLI:
1299
2232
 
1300
2233
  ```bash
1301
- zenml connect --url https://... --api-key <API_KEY>
2234
+ zenml login https://... --api-key
1302
2235
  ```
1303
2236
 
1304
2237
  or by setting the `ZENML_STORE_URL` and `ZENML_STORE_API_KEY` environment
1305
- variables when you set up your ZenML client for the first time:
2238
+ variables when you set up your ZenML client for the first time:
1306
2239
 
1307
2240
  ```bash
1308
2241
  export ZENML_STORE_URL=https://...
1309
2242
  export ZENML_STORE_API_KEY=<API_KEY>
1310
2243
  ```
1311
2244
 
2245
+ You don't need to run `zenml login` after setting these two environment
2246
+ variables and can start interacting with your server right away.
2247
+
1312
2248
  To see all the service accounts you've created and their API keys, use the
1313
2249
  following commands:
1314
2250
 
@@ -1368,82 +2304,202 @@ command:
1368
2304
  zenml service-account api-key <SERVICE_ACCOUNT_NAME> delete <API_KEY_NAME>
1369
2305
  ```
1370
2306
 
1371
- Deploying ZenML to the cloud
1372
- ----------------------------
2307
+ Managing Code Repositories
2308
+ --------------------------
2309
+
2310
+ [Code repositories](https://docs.zenml.io/user-guide/production-guide/connect-code-repository)
2311
+ enable ZenML to keep track of the code version that you use for your pipeline
2312
+ runs. Additionally, running a pipeline which is tracked in a registered code
2313
+ repository can decrease the time it takes Docker to build images for
2314
+ containerized stack components.
1373
2315
 
1374
- The ZenML CLI provides a simple way to deploy ZenML to the cloud. Simply run
2316
+ To register a code repository, use the following CLI
2317
+ command:
1375
2318
 
1376
- ```bash
1377
- zenml deploy
2319
+ ```shell
2320
+ zenml code-repository register <NAME> --type=<CODE_REPOSITORY_TYPE] \
2321
+ [--CODE_REPOSITORY_OPTIONS]
2322
+ ```
2323
+
2324
+ ZenML currently supports code repositories of type `github` and `gitlab`, but
2325
+ you can also use your custom code repository implementation by passing the
2326
+ type `custom` and a source of your repository class.
2327
+
2328
+ ```shell
2329
+ zenml code-repository register <NAME> --type=custom \
2330
+ --source=<CODE_REPOSITORY_SOURCE> [--CODE_REPOSITORY_OPTIONS]
2331
+ ```
2332
+
2333
+ The `CODE_REPOSITORY_OPTIONS` depend on the configuration necessary for the
2334
+ type of code repository that you're using.
2335
+
2336
+ If you want to list your registered code repositories, run:
2337
+
2338
+ ```shell
2339
+ zenml code-repository list
1378
2340
  ```
1379
2341
 
1380
- You will be prompted to provide a name for your deployment and details like what
1381
- cloud provider you want to deploy to, in addition to the username, password, and
1382
- email you want to set for the default user — and that's it! It creates the
1383
- database and any VPCs, permissions, and more that are needed.
2342
+ You can delete one of your registered code repositories like this:
1384
2343
 
1385
- In order to be able to run the deploy command, you should have your cloud
1386
- provider's CLI configured locally with permissions to create resources like
1387
- MySQL databases and networks.
2344
+ ```shell
2345
+ zenml code-repository delete <REPOSITORY_NAME_OR_ID>
2346
+ ```
1388
2347
 
1389
- Interacting with the ZenML Hub
2348
+ Building an image without Runs
1390
2349
  ------------------------------
1391
2350
 
1392
- The ZenML Hub is a central location for discovering and sharing third-party
1393
- ZenML code, such as custom integrations, components, steps, pipelines,
1394
- materializers, and more.
1395
- You can browse the ZenML Hub at [https://hub.zenml.io](https://hub.zenml.io).
2351
+ To build or run a pipeline from the CLI, you need to know the source path of
2352
+ your pipeline. Let's imagine you have defined your pipeline in a python file
2353
+ called `run.py` like this:
2354
+
2355
+ ```python
2356
+ from zenml import pipeline
2357
+
2358
+ @pipeline
2359
+ def my_pipeline(...):
2360
+ # Connect your pipeline steps here
2361
+ pass
2362
+ ```
2363
+
2364
+ The source path of your pipeline will be `run.my_pipeline`. In a generalized
2365
+ way, this will be `<MODULE_PATH>.<PIPELINE_FUNCTION_NAME>`. If the python file
2366
+ defining the pipeline is not in your current directory, the module path consists
2367
+ of the full path to the file, separated by dots, e.g.
2368
+ `some_directory.some_file.my_pipeline`.
2369
+
2370
+ To [build Docker images for your pipeline](https://docs.zenml.io/how-to/infrastructure-deployment/customize-docker-builds)
2371
+ without actually running the pipeline, use:
1396
2372
 
1397
- The ZenML CLI provides various commands to interact with the ZenML Hub:
2373
+ ```bash
2374
+ zenml pipeline build <PIPELINE_SOURCE_PATH>
2375
+ ```
2376
+
2377
+ To specify settings for the Docker builds, use the `--config/-c` option of the
2378
+ command. For more information about the structure of this configuration file,
2379
+ check out the `zenml.pipelines.base_pipeline.BasePipeline.build(...)` method.
2380
+
2381
+ ```bash
2382
+ zenml pipeline build <PIPELINE_SOURCE_PATH> --config=<PATH_TO_CONFIG_YAML>
2383
+ ```
2384
+
2385
+ If you want to build the pipeline for a stack other than your current active
2386
+ stack, use the `--stack` option.
2387
+
2388
+ ```bash
2389
+ zenml pipeline build <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
2390
+ ```
2391
+
2392
+ To run a pipeline that was previously registered, use:
2393
+
2394
+ ```bash
2395
+ zenml pipeline run <PIPELINE_SOURCE_PATH>
2396
+ ```
2397
+
2398
+ To specify settings for the pipeline, use the `--config/-c` option of the
2399
+ command. For more information about the structure of this configuration file,
2400
+ check out the `zenml.pipelines.base_pipeline.BasePipeline.run(...)` method.
2401
+
2402
+ ```bash
2403
+ zenml pipeline run <PIPELINE_SOURCE_PATH> --config=<PATH_TO_CONFIG_YAML>
2404
+ ```
2405
+
2406
+ If you want to run the pipeline on a stack different than your current active
2407
+ stack, use the `--stack` option.
2408
+
2409
+ ```bash
2410
+ zenml pipeline run <PIPELINE_SOURCE_PATH> --stack=<STACK_ID_OR_NAME>
2411
+ ```
2412
+
2413
+ If you want to create a run template based on your pipeline that can later be used to trigger a run either from the dashboard or through an HTTP request:
2414
+
2415
+ ```bash
2416
+ zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
2417
+ --name=<TEMPLATE_NAME>
2418
+
2419
+ To specify a config file, use the `--config/-c` option. If you would like to use a different stack than the active one, use the `--stack` option.
1398
2420
 
1399
- - Listing all plugins available on the Hub:
1400
2421
  ```bash
1401
- zenml hub list
2422
+ zenml pipeline create-run-template <PIPELINE_SOURCE_PATH> \
2423
+ --name=<TEMPLATE_NAME> \
2424
+ --config=<PATH_TO_CONFIG_YAML> \
2425
+ --stack=<STACK_ID_OR_NAME>
1402
2426
  ```
1403
2427
 
1404
- - Installing a Hub plugin:
2428
+ Tagging your resources with ZenML
2429
+ ---------------------------------
2430
+
2431
+ When you are using ZenML, you can [use tags to organize and categorize your
2432
+ assets](https://docs.zenml.io/how-to/handle-data-artifacts/tagging).
2433
+ This way, you can streamline your workflows and enhance the discoverability of
2434
+ your resources more easily.
2435
+
2436
+ Currently, you can use tags with artifacts, models and their versions:
2437
+
1405
2438
  ```bash
1406
- zenml hub install
2439
+ # Tag the artifact
2440
+ zenml artifact update ARTIFACT_NAME -t TAG_NAME
2441
+
2442
+ # Tag the artifact version
2443
+ zenml artifact version update ARTIFACT_NAME ARTIFACT_VERSION -t TAG_NAME
2444
+
2445
+ # Tag an existing model
2446
+ zenml model update MODEL_NAME --tag TAG_NAME
2447
+
2448
+ # Tag a specific model version
2449
+ zenml model version update MODEL_NAME VERSION_NAME --tag TAG_NAME
1407
2450
  ```
1408
- Installed plugins can be imported via `from zenml.hub.<plugin_name> import ...`.
1409
2451
 
2452
+ Besides these interactions, you can also create a new tag by using the
2453
+ `register` command:
1410
2454
 
1411
- - Uninstalling a Hub plugin:
1412
2455
  ```bash
1413
- zenml hub uninstall
2456
+ zenml tag register -n TAG_NAME [-c COLOR]
1414
2457
  ```
1415
2458
 
1416
- - Cloning the source code of a Hub plugin (without installing it):
2459
+ If you would like to list all the tags that you have, you can use the command:
2460
+
1417
2461
  ```bash
1418
- zenml hub clone
2462
+ zenml tag list
1419
2463
  ```
1420
- This is useful, e.g., for extending an existing plugin or for getting the
1421
- examples of a plugin.
1422
2464
 
1423
- - Submitting/contributing a plugin to the Hub (requires login, see below):
2465
+ To update the properties of a specific tag, you can use the `update` subcommand:
2466
+
1424
2467
  ```bash
1425
- zenml hub submit
2468
+ zenml tag update TAG_NAME_OR_ID [-n NEW_NAME] [-c NEW_COLOR]
1426
2469
  ```
1427
- If you are unsure about which arguments you need to set, you can run the
1428
- command in interactive mode:
2470
+
2471
+ Finally, in order to delete a tag, you can execute:
2472
+
1429
2473
  ```bash
1430
- zenml hub submit --interactive
2474
+ zenml tag delete TAG_NAME_OR_ID
1431
2475
  ```
1432
- This will ask for and validate inputs one at a time.
1433
2476
 
1434
- - Logging in to the Hub:
2477
+ Managing the Global Configuration
2478
+ ---------------------------------
2479
+
2480
+ The ZenML global configuration CLI commands cover options such as enabling or
2481
+ disabling the collection of anonymous usage statistics, changing the logging
2482
+ verbosity.
2483
+
2484
+ In order to help us better understand how the community uses ZenML, the library
2485
+ reports anonymized usage statistics. You can always opt out by using the CLI
2486
+ command:
2487
+
1435
2488
  ```bash
1436
- zenml hub login
2489
+ zenml analytics opt-out
1437
2490
  ```
1438
2491
 
1439
- - Logging out of the Hub:
2492
+ If you want to opt back in, use the following command:
2493
+
1440
2494
  ```bash
1441
- zenml hub logout
2495
+ zenml analytics opt-in
1442
2496
  ```
1443
2497
 
1444
- - Viewing the build logs of a plugin you submitted to the Hub:
2498
+ The verbosity of the ZenML client output can be configured using the
2499
+ ``zenml logging`` command. For example, to set the verbosity to DEBUG, run:
2500
+
1445
2501
  ```bash
1446
- zenml hub logs
2502
+ zenml logging set-verbosity DEBUG
1447
2503
  ```
1448
2504
  """
1449
2505
 
@@ -1456,8 +2512,8 @@ from zenml.cli.code_repository import * # noqa
1456
2512
  from zenml.cli.config import * # noqa
1457
2513
  from zenml.cli.downgrade import * # noqa
1458
2514
  from zenml.cli.feature import * # noqa
1459
- from zenml.cli.hub import * # noqa
1460
2515
  from zenml.cli.integration import * # noqa
2516
+ from zenml.cli.login import *
1461
2517
  from zenml.cli.model import * # noqa
1462
2518
  from zenml.cli.model_registry import * # noqa
1463
2519
  from zenml.cli.pipeline import * # noqa
@@ -1468,7 +2524,6 @@ from zenml.cli.service_accounts import * # noqa
1468
2524
  from zenml.cli.service_connectors import * # noqa
1469
2525
  from zenml.cli.stack import * # noqa
1470
2526
  from zenml.cli.stack_components import * # noqa
1471
- from zenml.cli.stack_recipes import * # noqa
1472
2527
  from zenml.cli.user_management import * # noqa
1473
2528
  from zenml.cli.workspace import * # noqa
1474
2529
  from zenml.cli.tag import * # noqa