zenml-nightly 0.68.1.dev20241103__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +77 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/artifacts/external_artifact.py +2 -1
  6. zenml/artifacts/{load_directory_materializer.py → preexisting_data_materializer.py} +8 -9
  7. zenml/artifacts/utils.py +139 -80
  8. zenml/cli/base.py +4 -4
  9. zenml/cli/model.py +1 -6
  10. zenml/cli/stack.py +1 -0
  11. zenml/client.py +29 -74
  12. zenml/config/server_config.py +17 -1
  13. zenml/constants.py +2 -7
  14. zenml/data_validators/base_data_validator.py +2 -2
  15. zenml/enums.py +20 -4
  16. zenml/exceptions.py +4 -0
  17. zenml/integrations/__init__.py +3 -1
  18. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
  19. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  20. zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
  21. zenml/integrations/constants.py +1 -0
  22. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
  23. zenml/integrations/evidently/__init__.py +1 -1
  24. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  25. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +8 -12
  26. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +17 -18
  27. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +2 -5
  28. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +17 -18
  29. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -3
  30. zenml/integrations/langchain/__init__.py +2 -1
  31. zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
  32. zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
  33. zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
  34. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
  35. zenml/integrations/openai/__init__.py +1 -1
  36. zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
  37. zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
  38. zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
  39. zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
  40. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
  41. zenml/integrations/tensorflow/materializers/keras_materializer.py +11 -22
  42. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +8 -15
  43. zenml/integrations/vllm/__init__.py +50 -0
  44. zenml/integrations/vllm/flavors/__init__.py +21 -0
  45. zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
  46. zenml/integrations/vllm/model_deployers/__init__.py +19 -0
  47. zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
  48. zenml/integrations/vllm/services/__init__.py +19 -0
  49. zenml/integrations/vllm/services/vllm_deployment.py +206 -0
  50. zenml/integrations/whylogs/materializers/whylogs_materializer.py +11 -18
  51. zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
  52. zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
  53. zenml/materializers/base_materializer.py +68 -1
  54. zenml/metadata/lazy_load.py +20 -7
  55. zenml/model/model.py +17 -64
  56. zenml/model/utils.py +5 -0
  57. zenml/models/__init__.py +0 -12
  58. zenml/models/v2/base/filter.py +121 -8
  59. zenml/models/v2/core/artifact_version.py +42 -7
  60. zenml/models/v2/core/model_version.py +26 -5
  61. zenml/models/v2/core/pipeline_run.py +25 -6
  62. zenml/models/v2/core/run_metadata.py +2 -217
  63. zenml/models/v2/core/step_run.py +62 -24
  64. zenml/orchestrators/base_orchestrator.py +12 -1
  65. zenml/orchestrators/input_utils.py +44 -19
  66. zenml/orchestrators/step_launcher.py +4 -3
  67. zenml/orchestrators/step_run_utils.py +19 -15
  68. zenml/orchestrators/step_runner.py +25 -14
  69. zenml/orchestrators/utils.py +45 -26
  70. zenml/stack/flavor.py +9 -5
  71. zenml/stack_deployments/aws_stack_deployment.py +23 -6
  72. zenml/stack_deployments/azure_stack_deployment.py +28 -5
  73. zenml/stack_deployments/gcp_stack_deployment.py +25 -8
  74. zenml/stack_deployments/stack_deployment.py +3 -5
  75. zenml/steps/base_step.py +1 -1
  76. zenml/steps/entrypoint_function_utils.py +3 -5
  77. zenml/steps/step_context.py +5 -2
  78. zenml/steps/utils.py +13 -2
  79. zenml/utils/callback_registry.py +71 -0
  80. zenml/utils/metadata_utils.py +335 -0
  81. zenml/zen_server/auth.py +221 -3
  82. zenml/zen_server/cache.py +208 -0
  83. zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
  84. zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
  86. zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
  87. zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
  88. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
  89. zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
  90. zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
  92. zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
  93. zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
  94. zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
  95. zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
  96. zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
  99. zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
  100. zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
  101. zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
  105. zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
  106. zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
  107. zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
  108. zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
  109. zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
  110. zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
  111. zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
  112. zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
  113. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
  114. zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
  115. zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
  116. zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
  117. zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
  118. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
  119. zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
  120. zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
  121. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
  123. zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
  124. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
  125. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  126. zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
  127. zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
  128. zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
  129. zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
  130. zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
  131. zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
  132. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
  133. zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
  135. zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
  136. zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
  137. zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
  138. zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
  139. zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
  140. zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
  141. zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
  142. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
  143. zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
  144. zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
  145. zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
  147. zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
  148. zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
  150. zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
  151. zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
  152. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
  153. zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
  154. zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
  155. zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
  156. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
  157. zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
  158. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
  159. zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
  160. zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
  161. zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
  162. zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
  163. zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
  164. zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
  165. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
  166. zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
  167. zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
  168. zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
  169. zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
  170. zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
  171. zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
  172. zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
  173. zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
  174. zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
  175. zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
  176. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
  177. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
  178. zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
  179. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
  180. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
  181. zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
  182. zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
  183. zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
  184. zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
  185. zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
  186. zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
  187. zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
  188. zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
  189. zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
  190. zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
  191. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
  192. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  193. zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
  194. zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
  195. zenml/zen_server/dashboard/index.html +7 -7
  196. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  197. zenml/zen_server/deploy/helm/README.md +2 -2
  198. zenml/zen_server/exceptions.py +2 -0
  199. zenml/zen_server/jwt.py +30 -13
  200. zenml/zen_server/rbac/endpoint_utils.py +43 -1
  201. zenml/zen_server/rbac/utils.py +0 -2
  202. zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
  203. zenml/zen_server/routers/auth_endpoints.py +134 -102
  204. zenml/zen_server/routers/logs_endpoints.py +66 -0
  205. zenml/zen_server/routers/workspaces_endpoints.py +3 -4
  206. zenml/zen_server/template_execution/utils.py +14 -16
  207. zenml/zen_server/utils.py +27 -0
  208. zenml/zen_server/zen_server_api.py +6 -3
  209. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  210. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  211. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  212. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  213. zenml/zen_stores/rest_zen_store.py +68 -64
  214. zenml/zen_stores/schemas/artifact_schemas.py +8 -1
  215. zenml/zen_stores/schemas/model_schemas.py +27 -3
  216. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
  217. zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
  218. zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
  219. zenml/zen_stores/schemas/step_run_schemas.py +18 -10
  220. zenml/zen_stores/sql_zen_store.py +283 -219
  221. zenml/zen_stores/zen_store_interface.py +15 -42
  222. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
  223. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +226 -203
  224. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
  225. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
  226. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
  227. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
  228. zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
  229. zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
  230. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
  231. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
  232. zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
  233. zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
  234. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  235. zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
  236. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
  237. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
  238. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
  239. zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
  240. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
  241. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
  242. zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
  243. zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
  244. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
  245. zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
  246. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
  247. zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
  248. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
  249. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
  250. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
  251. zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
  252. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
  253. zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
  254. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
  255. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
  256. zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
  257. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
  258. zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
  259. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
  260. zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
  261. zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
  262. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
  263. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
  264. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
@@ -18,17 +18,19 @@ from uuid import UUID
18
18
 
19
19
  from zenml.client import Client
20
20
  from zenml.config.step_configurations import Step
21
+ from zenml.enums import ArtifactSaveType, StepRunInputArtifactType
21
22
  from zenml.exceptions import InputResolutionError
22
23
  from zenml.utils import pagination_utils
23
24
 
24
25
  if TYPE_CHECKING:
25
- from zenml.models import ArtifactVersionResponse, PipelineRunResponse
26
+ from zenml.models import PipelineRunResponse
27
+ from zenml.models.v2.core.step_run import StepRunInputResponse
26
28
 
27
29
 
28
30
  def resolve_step_inputs(
29
31
  step: "Step",
30
32
  pipeline_run: "PipelineRunResponse",
31
- ) -> Tuple[Dict[str, "ArtifactVersionResponse"], List[UUID]]:
33
+ ) -> Tuple[Dict[str, "StepRunInputResponse"], List[UUID]]:
32
34
  """Resolves inputs for the current step.
33
35
 
34
36
  Args:
@@ -45,7 +47,8 @@ def resolve_step_inputs(
45
47
  The IDs of the input artifact versions and the IDs of parent steps of
46
48
  the current step.
47
49
  """
48
- from zenml.models import ArtifactVersionResponse, RunMetadataResponse
50
+ from zenml.models import ArtifactVersionResponse
51
+ from zenml.models.v2.core.step_run import StepRunInputResponse
49
52
 
50
53
  current_run_steps = {
51
54
  run_step.name: run_step
@@ -54,7 +57,7 @@ def resolve_step_inputs(
54
57
  )
55
58
  }
56
59
 
57
- input_artifacts: Dict[str, "ArtifactVersionResponse"] = {}
60
+ input_artifacts: Dict[str, StepRunInputResponse] = {}
58
61
  for name, input_ in step.spec.inputs.items():
59
62
  try:
60
63
  step_run = current_run_steps[input_.step_name]
@@ -64,22 +67,44 @@ def resolve_step_inputs(
64
67
  )
65
68
 
66
69
  try:
67
- artifact = step_run.outputs[input_.output_name]
70
+ outputs = step_run.outputs[input_.output_name]
68
71
  except KeyError:
69
72
  raise InputResolutionError(
70
- f"No output `{input_.output_name}` found for step "
73
+ f"No step output `{input_.output_name}` found for step "
71
74
  f"`{input_.step_name}`."
72
75
  )
73
76
 
74
- input_artifacts[name] = artifact
77
+ step_outputs = [
78
+ output
79
+ for output in outputs
80
+ if output.save_type == ArtifactSaveType.STEP_OUTPUT
81
+ ]
82
+ if len(step_outputs) > 2:
83
+ # This should never happen, there can only be a single regular step
84
+ # output for a name
85
+ raise InputResolutionError(
86
+ f"Too many step outputs for output `{input_.output_name}` of "
87
+ f"step `{input_.step_name}`."
88
+ )
89
+ elif len(step_outputs) == 0:
90
+ raise InputResolutionError(
91
+ f"No step output `{input_.output_name}` found for step "
92
+ f"`{input_.step_name}`."
93
+ )
94
+
95
+ input_artifacts[name] = StepRunInputResponse(
96
+ input_type=StepRunInputArtifactType.STEP_OUTPUT,
97
+ **step_outputs[0].model_dump(),
98
+ )
75
99
 
76
100
  for (
77
101
  name,
78
102
  external_artifact,
79
103
  ) in step.config.external_input_artifacts.items():
80
104
  artifact_version_id = external_artifact.get_artifact_version_id()
81
- input_artifacts[name] = Client().get_artifact_version(
82
- artifact_version_id
105
+ input_artifacts[name] = StepRunInputResponse(
106
+ input_type=StepRunInputArtifactType.EXTERNAL,
107
+ **Client().get_artifact_version(artifact_version_id).model_dump(),
83
108
  )
84
109
 
85
110
  for name, config_ in step.config.model_artifacts_or_metadata.items():
@@ -98,9 +123,7 @@ def resolve_step_inputs(
98
123
  ):
99
124
  # metadata values should go directly in parameters, as primitive types
100
125
  step.config.parameters[name] = (
101
- context_model_version.run_metadata[
102
- config_.metadata_name
103
- ].value
126
+ context_model_version.run_metadata[config_.metadata_name]
104
127
  )
105
128
  elif config_.artifact_name is None:
106
129
  err_msg = (
@@ -112,14 +135,15 @@ def resolve_step_inputs(
112
135
  config_.artifact_name, config_.artifact_version
113
136
  ):
114
137
  if config_.metadata_name is None:
115
- input_artifacts[name] = artifact_
138
+ input_artifacts[name] = StepRunInputResponse(
139
+ input_type=StepRunInputArtifactType.LAZY_LOADED,
140
+ **artifact_.model_dump(),
141
+ )
116
142
  elif config_.metadata_name:
117
143
  # metadata values should go directly in parameters, as primitive types
118
144
  try:
119
145
  step.config.parameters[name] = (
120
- artifact_.run_metadata[
121
- config_.metadata_name
122
- ].value
146
+ artifact_.run_metadata[config_.metadata_name]
123
147
  )
124
148
  except KeyError:
125
149
  err_msg = (
@@ -141,9 +165,10 @@ def resolve_step_inputs(
141
165
  for name, cll_ in step.config.client_lazy_loaders.items():
142
166
  value_ = cll_.evaluate()
143
167
  if isinstance(value_, ArtifactVersionResponse):
144
- input_artifacts[name] = value_
145
- elif isinstance(value_, RunMetadataResponse):
146
- step.config.parameters[name] = value_.value
168
+ input_artifacts[name] = StepRunInputResponse(
169
+ input_type=StepRunInputArtifactType.LAZY_LOADED,
170
+ **value_.model_dump(),
171
+ )
147
172
  else:
148
173
  step.config.parameters[name] = value_
149
174
 
@@ -33,13 +33,13 @@ from zenml.environment import get_run_environment_dict
33
33
  from zenml.logger import get_logger
34
34
  from zenml.logging import step_logging
35
35
  from zenml.models import (
36
- ArtifactVersionResponse,
37
36
  LogsRequest,
38
37
  PipelineDeploymentResponse,
39
38
  PipelineRunRequest,
40
39
  PipelineRunResponse,
41
40
  StepRunResponse,
42
41
  )
42
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
43
  from zenml.orchestrators import output_utils, publish_utils, step_run_utils
44
44
  from zenml.orchestrators import utils as orchestrator_utils
45
45
  from zenml.orchestrators.step_runner import StepRunner
@@ -422,7 +422,8 @@ class StepLauncher:
422
422
  )
423
423
  )
424
424
  environment = orchestrator_utils.get_config_environment_vars(
425
- deployment=self._deployment
425
+ pipeline_run_id=step_run_info.run_id,
426
+ step_run_id=step_run_info.step_run_id,
426
427
  )
427
428
  if last_retry:
428
429
  environment[ENV_ZENML_IGNORE_FAILURE_HOOK] = str(False)
@@ -442,7 +443,7 @@ class StepLauncher:
442
443
  pipeline_run: PipelineRunResponse,
443
444
  step_run: StepRunResponse,
444
445
  step_run_info: StepRunInfo,
445
- input_artifacts: Dict[str, ArtifactVersionResponse],
446
+ input_artifacts: Dict[str, StepRunInputResponse],
446
447
  output_artifact_uris: Dict[str, str],
447
448
  last_retry: bool,
448
449
  ) -> None:
@@ -14,12 +14,12 @@
14
14
  """Utilities for creating step runs."""
15
15
 
16
16
  from datetime import datetime
17
- from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
17
+ from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple
18
18
 
19
19
  from zenml.client import Client
20
20
  from zenml.config.step_configurations import ArtifactConfiguration, Step
21
21
  from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
22
- from zenml.enums import ExecutionStatus
22
+ from zenml.enums import ArtifactSaveType, ExecutionStatus
23
23
  from zenml.logger import get_logger
24
24
  from zenml.model.utils import link_artifact_version_to_model_version
25
25
  from zenml.models import (
@@ -104,6 +104,7 @@ class StepRunRequestFactory:
104
104
  input_name: artifact.id
105
105
  for input_name, artifact in input_artifacts.items()
106
106
  }
107
+
107
108
  request.inputs = input_artifact_ids
108
109
  request.parent_step_ids = parent_step_ids
109
110
 
@@ -142,8 +143,8 @@ class StepRunRequestFactory:
142
143
 
143
144
  request.original_step_run_id = cached_step_run.id
144
145
  request.outputs = {
145
- output_name: artifact.id
146
- for output_name, artifact in cached_step_run.outputs.items()
146
+ output_name: [artifact.id for artifact in artifacts]
147
+ for output_name, artifacts in cached_step_run.outputs.items()
147
148
  }
148
149
 
149
150
  request.status = ExecutionStatus.CACHED
@@ -551,7 +552,7 @@ def link_pipeline_run_to_model_version(
551
552
 
552
553
 
553
554
  def link_output_artifacts_to_model_version(
554
- artifacts: Dict[str, ArtifactVersionResponse],
555
+ artifacts: Dict[str, List[ArtifactVersionResponse]],
555
556
  output_configurations: Mapping[str, ArtifactConfiguration],
556
557
  model_version: ModelVersionResponse,
557
558
  ) -> None:
@@ -562,13 +563,16 @@ def link_output_artifacts_to_model_version(
562
563
  output_configurations: The output configurations for the step.
563
564
  model_version: The model version to link.
564
565
  """
565
- for output_name, output_artifact in artifacts.items():
566
- artifact_config = None
567
- if output_config := output_configurations.get(output_name, None):
568
- artifact_config = output_config.artifact_config
569
-
570
- link_artifact_version_to_model_version(
571
- artifact_version=output_artifact,
572
- model_version=model_version,
573
- artifact_config=artifact_config,
574
- )
566
+ for output_name, output_artifacts in artifacts.items():
567
+ for output_artifact in output_artifacts:
568
+ artifact_config = None
569
+ if output_artifact.save_type == ArtifactSaveType.STEP_OUTPUT and (
570
+ output_config := output_configurations.get(output_name, None)
571
+ ):
572
+ artifact_config = output_config.artifact_config
573
+
574
+ link_artifact_version_to_model_version(
575
+ artifact_version=output_artifact,
576
+ model_version=model_version,
577
+ artifact_config=artifact_config,
578
+ )
@@ -28,7 +28,8 @@ from typing import (
28
28
  )
29
29
 
30
30
  from zenml.artifacts.unmaterialized_artifact import UnmaterializedArtifact
31
- from zenml.artifacts.utils import save_artifact
31
+ from zenml.artifacts.utils import _store_artifact_data_and_prepare_request
32
+ from zenml.client import Client
32
33
  from zenml.config.step_configurations import StepConfiguration
33
34
  from zenml.config.step_run_info import StepRunInfo
34
35
  from zenml.constants import (
@@ -36,10 +37,12 @@ from zenml.constants import (
36
37
  ENV_ZENML_IGNORE_FAILURE_HOOK,
37
38
  handle_bool_env_var,
38
39
  )
40
+ from zenml.enums import ArtifactSaveType
39
41
  from zenml.exceptions import StepInterfaceError
40
42
  from zenml.logger import get_logger
41
43
  from zenml.logging.step_logging import StepLogsStorageContext, redirected
42
44
  from zenml.materializers.base_materializer import BaseMaterializer
45
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
46
  from zenml.orchestrators.publish_utils import (
44
47
  publish_step_run_metadata,
45
48
  publish_successful_step_run,
@@ -98,7 +101,7 @@ class StepRunner:
98
101
  self,
99
102
  pipeline_run: "PipelineRunResponse",
100
103
  step_run: "StepRunResponse",
101
- input_artifacts: Dict[str, "ArtifactVersionResponse"],
104
+ input_artifacts: Dict[str, StepRunInputResponse],
102
105
  output_artifact_uris: Dict[str, str],
103
106
  step_run_info: StepRunInfo,
104
107
  ) -> None:
@@ -153,7 +156,7 @@ class StepRunner:
153
156
 
154
157
  # Initialize the step context singleton
155
158
  StepContext._clear()
156
- StepContext(
159
+ step_context = StepContext(
157
160
  pipeline_run=pipeline_run,
158
161
  step_run=step_run,
159
162
  output_materializers=output_materializers,
@@ -241,11 +244,16 @@ class StepRunner:
241
244
  from zenml.orchestrators import step_run_utils
242
245
 
243
246
  step_run_utils.link_output_artifacts_to_model_version(
244
- artifacts=output_artifacts,
247
+ artifacts={
248
+ k: [v] for k, v in output_artifacts.items()
249
+ },
245
250
  output_configurations=step_run.config.outputs,
246
251
  model_version=model_version,
247
252
  )
248
253
  finally:
254
+ step_context._cleanup_registry.execute_callbacks(
255
+ raise_on_exception=False
256
+ )
249
257
  StepContext._clear() # Remove the step context singleton
250
258
 
251
259
  # Update the status and output artifacts of the step run.
@@ -299,7 +307,7 @@ class StepRunner:
299
307
  self,
300
308
  args: List[str],
301
309
  annotations: Dict[str, Any],
302
- input_artifacts: Dict[str, "ArtifactVersionResponse"],
310
+ input_artifacts: Dict[str, StepRunInputResponse],
303
311
  ) -> Dict[str, Any]:
304
312
  """Parses the inputs for a step entrypoint function.
305
313
 
@@ -531,7 +539,7 @@ class StepRunner:
531
539
  The IDs of the published output artifacts.
532
540
  """
533
541
  step_context = get_step_context()
534
- output_artifacts: Dict[str, "ArtifactVersionResponse"] = {}
542
+ artifact_requests = []
535
543
 
536
544
  for output_name, return_value in output_data.items():
537
545
  data_type = type(return_value)
@@ -592,22 +600,25 @@ class StepRunner:
592
600
  # Get full set of tags
593
601
  tags = step_context.get_output_tags(output_name)
594
602
 
595
- artifact = save_artifact(
603
+ artifact_request = _store_artifact_data_and_prepare_request(
596
604
  name=artifact_name,
597
605
  data=return_value,
598
- materializer=materializer_class,
606
+ materializer_class=materializer_class,
599
607
  uri=uri,
600
- extract_metadata=artifact_metadata_enabled,
601
- include_visualizations=artifact_visualization_enabled,
608
+ store_metadata=artifact_metadata_enabled,
609
+ store_visualizations=artifact_visualization_enabled,
602
610
  has_custom_name=has_custom_name,
603
611
  version=version,
604
612
  tags=tags,
605
- user_metadata=user_metadata,
606
- manual_save=False,
613
+ save_type=ArtifactSaveType.STEP_OUTPUT,
614
+ metadata=user_metadata,
607
615
  )
608
- output_artifacts[output_name] = artifact
616
+ artifact_requests.append(artifact_request)
609
617
 
610
- return output_artifacts
618
+ responses = Client().zen_store.batch_create_artifact_versions(
619
+ artifact_requests
620
+ )
621
+ return dict(zip(output_data.keys(), responses))
611
622
 
612
623
  def load_and_run_hook(
613
624
  self,
@@ -28,16 +28,16 @@ from zenml.constants import (
28
28
  ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING,
29
29
  ENV_ZENML_SERVER,
30
30
  ENV_ZENML_STORE_PREFIX,
31
- PIPELINE_API_TOKEN_EXPIRES_MINUTES,
32
31
  )
33
32
  from zenml.enums import AuthScheme, StackComponentType, StoreType
34
33
  from zenml.logger import get_logger
35
34
  from zenml.stack import StackComponent
36
35
  from zenml.utils.string_utils import format_name_template
37
36
 
37
+ logger = get_logger(__name__)
38
+
38
39
  if TYPE_CHECKING:
39
40
  from zenml.artifact_stores.base_artifact_store import BaseArtifactStore
40
- from zenml.models import PipelineDeploymentResponse
41
41
 
42
42
 
43
43
  def get_orchestrator_run_name(pipeline_name: str) -> str:
@@ -80,16 +80,23 @@ def is_setting_enabled(
80
80
 
81
81
 
82
82
  def get_config_environment_vars(
83
- deployment: Optional["PipelineDeploymentResponse"] = None,
83
+ schedule_id: Optional[UUID] = None,
84
+ pipeline_run_id: Optional[UUID] = None,
85
+ step_run_id: Optional[UUID] = None,
84
86
  ) -> Dict[str, str]:
85
87
  """Gets environment variables to set for mirroring the active config.
86
88
 
87
- If a pipeline deployment is given, the environment variables will be set to
88
- include a newly generated API token valid for the duration of the pipeline
89
- run instead of the API token from the global config.
89
+ If a schedule ID, pipeline run ID or step run ID is given, and the current
90
+ client is not authenticated to a server with an API key, the environment
91
+ variables will be updated to include a newly generated workload API token
92
+ that will be valid for the duration of the schedule, pipeline run, or step
93
+ run instead of the current API token used to authenticate the client.
90
94
 
91
95
  Args:
92
- deployment: Optional deployment to use for the environment variables.
96
+ schedule_id: Optional schedule ID to use to generate a new API token.
97
+ pipeline_run_id: Optional pipeline run ID to use to generate a new API
98
+ token.
99
+ step_run_id: Optional step run ID to use to generate a new API token.
93
100
 
94
101
  Returns:
95
102
  Environment variable dict.
@@ -107,34 +114,46 @@ def get_config_environment_vars(
107
114
  ):
108
115
  credentials_store = get_credentials_store()
109
116
  url = global_config.store_configuration.url
110
- api_key = credentials_store.get_api_key(url)
111
117
  api_token = credentials_store.get_token(url, allow_expired=False)
112
- if api_key:
113
- environment_vars[ENV_ZENML_STORE_PREFIX + "API_KEY"] = api_key
114
- elif deployment:
115
- # When connected to an authenticated ZenML server, if a pipeline
116
- # deployment is supplied, we need to fetch an API token that will be
117
- # valid for the duration of the pipeline run.
118
+ if schedule_id or pipeline_run_id or step_run_id:
119
+ # When connected to an authenticated ZenML server, if a schedule ID,
120
+ # pipeline run ID or step run ID is supplied, we need to fetch a new
121
+ # workload API token scoped to the schedule, pipeline run or step
122
+ # run.
118
123
  assert isinstance(global_config.zen_store, RestZenStore)
119
- pipeline_id: Optional[UUID] = None
120
- if deployment.pipeline:
121
- pipeline_id = deployment.pipeline.id
122
- schedule_id: Optional[UUID] = None
123
- expires_minutes: Optional[int] = PIPELINE_API_TOKEN_EXPIRES_MINUTES
124
- if deployment.schedule:
125
- schedule_id = deployment.schedule.id
126
- # If a schedule is given, this is a long running pipeline that
127
- # should not have an API token that expires.
128
- expires_minutes = None
124
+
125
+ # If only a schedule is given, the pipeline run credentials will
126
+ # be valid for the entire duration of the schedule.
127
+ api_key = credentials_store.get_api_key(url)
128
+ if not api_key and not pipeline_run_id and not step_run_id:
129
+ logger.warning(
130
+ "An API token without an expiration time will be generated "
131
+ "and used to run this pipeline on a schedule. This is very "
132
+ "insecure because the API token will be valid for the "
133
+ "entire lifetime of the schedule and can be used to access "
134
+ "your user account if accidentally leaked. When deploying "
135
+ "a pipeline on a schedule, it is strongly advised to use a "
136
+ "service account API key to authenticate to the ZenML "
137
+ "server instead of your regular user account. For more "
138
+ "information, see "
139
+ "https://docs.zenml.io/how-to/connecting-to-zenml/connect-with-a-service-account"
140
+ )
141
+
142
+ # The schedule, pipeline run or step run credentials are scoped to
143
+ # the schedule, pipeline run or step run and will only be valid for
144
+ # the duration of the schedule/pipeline run/step run.
129
145
  new_api_token = global_config.zen_store.get_api_token(
130
- pipeline_id=pipeline_id,
131
146
  schedule_id=schedule_id,
132
- expires_minutes=expires_minutes,
147
+ pipeline_run_id=pipeline_run_id,
148
+ step_run_id=step_run_id,
133
149
  )
150
+
134
151
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
135
152
  new_api_token
136
153
  )
137
154
  elif api_token:
155
+ # For all other cases, the pipeline run environment is configured
156
+ # with the current access token.
138
157
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
139
158
  api_token.access_token
140
159
  )
zenml/stack/flavor.py CHANGED
@@ -16,6 +16,7 @@
16
16
  from abc import abstractmethod
17
17
  from typing import Any, Dict, Optional, Type, cast
18
18
 
19
+ from zenml.client import Client
19
20
  from zenml.enums import StackComponentType
20
21
  from zenml.models import (
21
22
  FlavorRequest,
@@ -146,9 +147,6 @@ class Flavor:
146
147
  Returns:
147
148
  The model.
148
149
  """
149
- from zenml.client import Client
150
-
151
- client = Client()
152
150
  connector_requirements = self.service_connector_requirements
153
151
  connector_type = (
154
152
  connector_requirements.connector_type
@@ -165,10 +163,16 @@ class Flavor:
165
163
  if connector_requirements
166
164
  else None
167
165
  )
166
+ user = None
167
+ workspace = None
168
+ if is_custom:
169
+ user = Client().active_user.id
170
+ workspace = Client().active_workspace.id
171
+
168
172
  model_class = FlavorRequest if is_custom else InternalFlavorRequest
169
173
  model = model_class(
170
- user=client.active_user.id if is_custom else None,
171
- workspace=client.active_workspace.id if is_custom else None,
174
+ user=user,
175
+ workspace=workspace,
172
176
  name=self.name,
173
177
  type=self.type,
174
178
  source=source_utils.resolve(self.__class__).import_path,
@@ -71,7 +71,8 @@ of any potential costs:
71
71
 
72
72
  - An S3 bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/s3).
73
73
  - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
74
- - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker).
74
+ - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
75
+ as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
75
76
  - An IAM user and IAM role with the minimum necessary permissions to access the
76
77
  above resources.
77
78
  - An AWS access key used to give access to ZenML to connect to the above
@@ -257,13 +258,29 @@ console.
257
258
 
258
259
  config: Optional[str] = None
259
260
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
260
- config = f"""module "zenml_stack" {{
261
- source = "zenml-io/zenml-stack/aws"
261
+ config = f"""terraform {{
262
+ required_providers {{
263
+ aws = {{
264
+ source = "hashicorp/aws"
265
+ }}
266
+ zenml = {{
267
+ source = "zenml-io/zenml"
268
+ }}
269
+ }}
270
+ }}
262
271
 
272
+ provider "aws" {{
263
273
  region = "{self.location or "eu-central-1"}"
264
- zenml_server_url = "{self.zenml_server_url}"
265
- zenml_api_key = ""
266
- zenml_api_token = "{self.zenml_server_api_token}"
274
+ }}
275
+
276
+ provider "zenml" {{
277
+ server_url = "{self.zenml_server_url}"
278
+ api_token = "{self.zenml_server_api_token}"
279
+ }}
280
+
281
+ module "zenml_stack" {{
282
+ source = "zenml-io/zenml-stack/aws"
283
+
267
284
  zenml_stack_name = "{self.stack_name}"
268
285
  zenml_stack_deployment = "{self.deployment_type}"
269
286
  }}
@@ -259,14 +259,37 @@ ZenML's access to your Azure subscription.
259
259
  The configuration or script to deploy the ZenML stack to the
260
260
  specified cloud provider.
261
261
  """
262
- config = f"""module "zenml_stack" {{
262
+ config = f"""terraform {{
263
+ required_providers {{
264
+ azurerm = {{
265
+ source = "hashicorp/azurerm"
266
+ }}
267
+ azuread = {{
268
+ source = "hashicorp/azuread"
269
+ }}
270
+ zenml = {{
271
+ source = "zenml-io/zenml"
272
+ }}
273
+ }}
274
+ }}
275
+
276
+ provider "azurerm" {{
277
+ features {{
278
+ resource_group {{
279
+ prevent_deletion_if_contains_resources = false
280
+ }}
281
+ }}
282
+ }}
283
+
284
+ provider "zenml" {{
285
+ server_url = "{self.zenml_server_url}"
286
+ api_token = "{self.zenml_server_api_token}"
287
+ }}
288
+
289
+ module "zenml_stack" {{
263
290
  source = "zenml-io/zenml-stack/azure"
264
291
 
265
292
  location = "{self.location or "eastus"}"
266
- orchestrator = "azureml"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
293
  zenml_stack_name = "{self.stack_name}"
271
294
  zenml_stack_deployment = "{self.deployment_type}"
272
295
  }}
@@ -70,11 +70,12 @@ and are aware of any potential costs:
70
70
 
71
71
  - A GCS bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/gcp).
72
72
  - A Google Artifact Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/gcp).
73
- - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex).
73
+ - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
74
+ and as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/vertex).
74
75
  - GCP Cloud Build registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builders/gcp).
75
76
  - A GCP Service Account with the minimum necessary permissions to access the
76
77
  above resources.
77
- - An GCP Service Account access key used to give access to ZenML to connect to
78
+ - A GCP Service Account access key used to give access to ZenML to connect to
78
79
  the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/gcp-service-connector).
79
80
 
80
81
  The Deployment Manager deployment will automatically create a GCP Service
@@ -259,14 +260,30 @@ GCP project and to clean up the resources created by the stack by using
259
260
  )
260
261
 
261
262
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
262
- config = f"""module "zenml_stack" {{
263
+ config = f"""terraform {{
264
+ required_providers {{
265
+ google = {{
266
+ source = "hashicorp/google"
267
+ }}
268
+ zenml = {{
269
+ source = "zenml-io/zenml"
270
+ }}
271
+ }}
272
+ }}
273
+
274
+ provider "google" {{
275
+ region = "{self.location or "europe-west3"}"
276
+ project = your GCP project name
277
+ }}
278
+
279
+ provider "zenml" {{
280
+ server_url = "{self.zenml_server_url}"
281
+ api_token = "{self.zenml_server_api_token}"
282
+ }}
283
+
284
+ module "zenml_stack" {{
263
285
  source = "zenml-io/zenml-stack/gcp"
264
286
 
265
- project_id = "my-gcp-project"
266
- region = "{self.location or "europe-west3"}"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
287
  zenml_stack_name = "{self.stack_name}"
271
288
  zenml_stack_deployment = "{self.deployment_type}"
272
289
  }}
@@ -219,16 +219,14 @@ class ZenMLCloudStackDeployment(BaseModel):
219
219
  if stack.labels.get("zenml:deployment") != self.deployment_type:
220
220
  continue
221
221
 
222
- artifact_store = stack.components[
223
- StackComponentType.ARTIFACT_STORE
224
- ][0]
222
+ orchestrator = stack.components[StackComponentType.ORCHESTRATOR][0]
225
223
 
226
- if not artifact_store.connector:
224
+ if not orchestrator.connector:
227
225
  continue
228
226
 
229
227
  return DeployedStack(
230
228
  stack=stack,
231
- service_connector=artifact_store.connector,
229
+ service_connector=orchestrator.connector,
232
230
  )
233
231
 
234
232
  return None