zenml-nightly 0.68.1.dev20241106__py3-none-any.whl → 0.70.0.dev20241116__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +77 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/artifacts/external_artifact.py +2 -1
  6. zenml/artifacts/utils.py +138 -79
  7. zenml/cli/base.py +4 -4
  8. zenml/cli/model.py +1 -6
  9. zenml/cli/stack.py +1 -0
  10. zenml/client.py +29 -74
  11. zenml/config/server_config.py +17 -1
  12. zenml/constants.py +2 -7
  13. zenml/data_validators/base_data_validator.py +2 -2
  14. zenml/enums.py +20 -4
  15. zenml/exceptions.py +4 -0
  16. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
  17. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  18. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
  19. zenml/integrations/evidently/__init__.py +1 -1
  20. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  21. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
  22. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
  23. zenml/integrations/vllm/services/vllm_deployment.py +16 -7
  24. zenml/metadata/lazy_load.py +20 -7
  25. zenml/model/model.py +17 -64
  26. zenml/model/utils.py +5 -0
  27. zenml/models/__init__.py +0 -12
  28. zenml/models/v2/base/filter.py +121 -8
  29. zenml/models/v2/core/artifact_version.py +42 -7
  30. zenml/models/v2/core/model_version.py +26 -5
  31. zenml/models/v2/core/pipeline_run.py +25 -6
  32. zenml/models/v2/core/run_metadata.py +2 -217
  33. zenml/models/v2/core/step_run.py +62 -24
  34. zenml/orchestrators/base_orchestrator.py +12 -1
  35. zenml/orchestrators/input_utils.py +44 -19
  36. zenml/orchestrators/step_launcher.py +4 -3
  37. zenml/orchestrators/step_run_utils.py +19 -15
  38. zenml/orchestrators/step_runner.py +21 -13
  39. zenml/orchestrators/utils.py +45 -26
  40. zenml/stack_deployments/aws_stack_deployment.py +23 -6
  41. zenml/stack_deployments/azure_stack_deployment.py +28 -5
  42. zenml/stack_deployments/gcp_stack_deployment.py +25 -8
  43. zenml/stack_deployments/stack_deployment.py +3 -5
  44. zenml/steps/base_step.py +1 -1
  45. zenml/steps/entrypoint_function_utils.py +3 -5
  46. zenml/steps/step_context.py +3 -2
  47. zenml/steps/utils.py +13 -2
  48. zenml/utils/metadata_utils.py +335 -0
  49. zenml/zen_server/auth.py +221 -3
  50. zenml/zen_server/cache.py +208 -0
  51. zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
  52. zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
  53. zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
  54. zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
  55. zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
  56. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
  57. zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
  58. zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
  59. zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
  60. zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
  61. zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
  62. zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
  63. zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
  64. zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
  65. zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
  66. zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
  67. zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
  68. zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
  69. zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
  71. zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
  72. zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
  73. zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
  74. zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
  75. zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
  76. zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
  77. zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
  78. zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
  79. zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
  80. zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
  81. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
  82. zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
  83. zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
  84. zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
  86. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
  87. zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
  88. zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
  89. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
  90. zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
  92. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
  93. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  94. zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
  95. zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
  96. zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
  99. zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
  100. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
  101. zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
  105. zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
  106. zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
  107. zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
  108. zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
  109. zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
  110. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
  111. zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
  112. zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
  113. zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
  114. zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
  115. zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
  116. zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
  117. zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
  118. zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
  119. zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
  120. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
  121. zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
  122. zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
  123. zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
  124. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
  125. zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
  126. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
  127. zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
  128. zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
  129. zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
  130. zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
  133. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
  134. zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
  135. zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
  136. zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
  137. zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
  138. zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
  139. zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
  142. zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
  143. zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
  144. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
  145. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
  146. zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
  147. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
  148. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
  149. zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
  150. zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
  151. zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
  152. zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
  153. zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
  155. zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
  156. zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
  157. zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
  158. zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
  159. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
  160. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  161. zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
  162. zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
  163. zenml/zen_server/dashboard/index.html +7 -7
  164. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  165. zenml/zen_server/deploy/helm/README.md +2 -2
  166. zenml/zen_server/exceptions.py +2 -0
  167. zenml/zen_server/jwt.py +30 -13
  168. zenml/zen_server/rbac/endpoint_utils.py +43 -1
  169. zenml/zen_server/rbac/utils.py +0 -2
  170. zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
  171. zenml/zen_server/routers/auth_endpoints.py +134 -102
  172. zenml/zen_server/routers/logs_endpoints.py +66 -0
  173. zenml/zen_server/routers/workspaces_endpoints.py +3 -4
  174. zenml/zen_server/template_execution/utils.py +14 -16
  175. zenml/zen_server/utils.py +27 -0
  176. zenml/zen_server/zen_server_api.py +6 -3
  177. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  178. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  179. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  180. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  181. zenml/zen_stores/rest_zen_store.py +68 -64
  182. zenml/zen_stores/schemas/artifact_schemas.py +8 -1
  183. zenml/zen_stores/schemas/model_schemas.py +27 -3
  184. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
  185. zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
  186. zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
  187. zenml/zen_stores/schemas/step_run_schemas.py +18 -10
  188. zenml/zen_stores/sql_zen_store.py +283 -219
  189. zenml/zen_stores/zen_store_interface.py +15 -42
  190. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/METADATA +2 -2
  191. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/RECORD +194 -179
  192. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
  193. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
  194. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
  195. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
  196. zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
  197. zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
  198. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
  199. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
  200. zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
  201. zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
  202. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  203. zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
  204. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
  205. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
  206. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
  207. zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
  208. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
  209. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
  210. zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
  211. zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
  212. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
  213. zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
  214. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
  215. zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
  216. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
  217. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
  218. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
  219. zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
  220. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
  221. zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
  222. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
  223. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
  224. zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
  225. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
  226. zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
  227. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
  228. zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
  229. zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
  230. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/LICENSE +0 -0
  231. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/WHEEL +0 -0
  232. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/entry_points.txt +0 -0
@@ -28,16 +28,16 @@ from zenml.constants import (
28
28
  ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING,
29
29
  ENV_ZENML_SERVER,
30
30
  ENV_ZENML_STORE_PREFIX,
31
- PIPELINE_API_TOKEN_EXPIRES_MINUTES,
32
31
  )
33
32
  from zenml.enums import AuthScheme, StackComponentType, StoreType
34
33
  from zenml.logger import get_logger
35
34
  from zenml.stack import StackComponent
36
35
  from zenml.utils.string_utils import format_name_template
37
36
 
37
+ logger = get_logger(__name__)
38
+
38
39
  if TYPE_CHECKING:
39
40
  from zenml.artifact_stores.base_artifact_store import BaseArtifactStore
40
- from zenml.models import PipelineDeploymentResponse
41
41
 
42
42
 
43
43
  def get_orchestrator_run_name(pipeline_name: str) -> str:
@@ -80,16 +80,23 @@ def is_setting_enabled(
80
80
 
81
81
 
82
82
  def get_config_environment_vars(
83
- deployment: Optional["PipelineDeploymentResponse"] = None,
83
+ schedule_id: Optional[UUID] = None,
84
+ pipeline_run_id: Optional[UUID] = None,
85
+ step_run_id: Optional[UUID] = None,
84
86
  ) -> Dict[str, str]:
85
87
  """Gets environment variables to set for mirroring the active config.
86
88
 
87
- If a pipeline deployment is given, the environment variables will be set to
88
- include a newly generated API token valid for the duration of the pipeline
89
- run instead of the API token from the global config.
89
+ If a schedule ID, pipeline run ID or step run ID is given, and the current
90
+ client is not authenticated to a server with an API key, the environment
91
+ variables will be updated to include a newly generated workload API token
92
+ that will be valid for the duration of the schedule, pipeline run, or step
93
+ run instead of the current API token used to authenticate the client.
90
94
 
91
95
  Args:
92
- deployment: Optional deployment to use for the environment variables.
96
+ schedule_id: Optional schedule ID to use to generate a new API token.
97
+ pipeline_run_id: Optional pipeline run ID to use to generate a new API
98
+ token.
99
+ step_run_id: Optional step run ID to use to generate a new API token.
93
100
 
94
101
  Returns:
95
102
  Environment variable dict.
@@ -107,34 +114,46 @@ def get_config_environment_vars(
107
114
  ):
108
115
  credentials_store = get_credentials_store()
109
116
  url = global_config.store_configuration.url
110
- api_key = credentials_store.get_api_key(url)
111
117
  api_token = credentials_store.get_token(url, allow_expired=False)
112
- if api_key:
113
- environment_vars[ENV_ZENML_STORE_PREFIX + "API_KEY"] = api_key
114
- elif deployment:
115
- # When connected to an authenticated ZenML server, if a pipeline
116
- # deployment is supplied, we need to fetch an API token that will be
117
- # valid for the duration of the pipeline run.
118
+ if schedule_id or pipeline_run_id or step_run_id:
119
+ # When connected to an authenticated ZenML server, if a schedule ID,
120
+ # pipeline run ID or step run ID is supplied, we need to fetch a new
121
+ # workload API token scoped to the schedule, pipeline run or step
122
+ # run.
118
123
  assert isinstance(global_config.zen_store, RestZenStore)
119
- pipeline_id: Optional[UUID] = None
120
- if deployment.pipeline:
121
- pipeline_id = deployment.pipeline.id
122
- schedule_id: Optional[UUID] = None
123
- expires_minutes: Optional[int] = PIPELINE_API_TOKEN_EXPIRES_MINUTES
124
- if deployment.schedule:
125
- schedule_id = deployment.schedule.id
126
- # If a schedule is given, this is a long running pipeline that
127
- # should not have an API token that expires.
128
- expires_minutes = None
124
+
125
+ # If only a schedule is given, the pipeline run credentials will
126
+ # be valid for the entire duration of the schedule.
127
+ api_key = credentials_store.get_api_key(url)
128
+ if not api_key and not pipeline_run_id and not step_run_id:
129
+ logger.warning(
130
+ "An API token without an expiration time will be generated "
131
+ "and used to run this pipeline on a schedule. This is very "
132
+ "insecure because the API token will be valid for the "
133
+ "entire lifetime of the schedule and can be used to access "
134
+ "your user account if accidentally leaked. When deploying "
135
+ "a pipeline on a schedule, it is strongly advised to use a "
136
+ "service account API key to authenticate to the ZenML "
137
+ "server instead of your regular user account. For more "
138
+ "information, see "
139
+ "https://docs.zenml.io/how-to/connecting-to-zenml/connect-with-a-service-account"
140
+ )
141
+
142
+ # The schedule, pipeline run or step run credentials are scoped to
143
+ # the schedule, pipeline run or step run and will only be valid for
144
+ # the duration of the schedule/pipeline run/step run.
129
145
  new_api_token = global_config.zen_store.get_api_token(
130
- pipeline_id=pipeline_id,
131
146
  schedule_id=schedule_id,
132
- expires_minutes=expires_minutes,
147
+ pipeline_run_id=pipeline_run_id,
148
+ step_run_id=step_run_id,
133
149
  )
150
+
134
151
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
135
152
  new_api_token
136
153
  )
137
154
  elif api_token:
155
+ # For all other cases, the pipeline run environment is configured
156
+ # with the current access token.
138
157
  environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
139
158
  api_token.access_token
140
159
  )
@@ -71,7 +71,8 @@ of any potential costs:
71
71
 
72
72
  - An S3 bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/s3).
73
73
  - An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
74
- - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker).
74
+ - Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
75
+ as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
75
76
  - An IAM user and IAM role with the minimum necessary permissions to access the
76
77
  above resources.
77
78
  - An AWS access key used to give access to ZenML to connect to the above
@@ -257,13 +258,29 @@ console.
257
258
 
258
259
  config: Optional[str] = None
259
260
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
260
- config = f"""module "zenml_stack" {{
261
- source = "zenml-io/zenml-stack/aws"
261
+ config = f"""terraform {{
262
+ required_providers {{
263
+ aws = {{
264
+ source = "hashicorp/aws"
265
+ }}
266
+ zenml = {{
267
+ source = "zenml-io/zenml"
268
+ }}
269
+ }}
270
+ }}
262
271
 
272
+ provider "aws" {{
263
273
  region = "{self.location or "eu-central-1"}"
264
- zenml_server_url = "{self.zenml_server_url}"
265
- zenml_api_key = ""
266
- zenml_api_token = "{self.zenml_server_api_token}"
274
+ }}
275
+
276
+ provider "zenml" {{
277
+ server_url = "{self.zenml_server_url}"
278
+ api_token = "{self.zenml_server_api_token}"
279
+ }}
280
+
281
+ module "zenml_stack" {{
282
+ source = "zenml-io/zenml-stack/aws"
283
+
267
284
  zenml_stack_name = "{self.stack_name}"
268
285
  zenml_stack_deployment = "{self.deployment_type}"
269
286
  }}
@@ -259,14 +259,37 @@ ZenML's access to your Azure subscription.
259
259
  The configuration or script to deploy the ZenML stack to the
260
260
  specified cloud provider.
261
261
  """
262
- config = f"""module "zenml_stack" {{
262
+ config = f"""terraform {{
263
+ required_providers {{
264
+ azurerm = {{
265
+ source = "hashicorp/azurerm"
266
+ }}
267
+ azuread = {{
268
+ source = "hashicorp/azuread"
269
+ }}
270
+ zenml = {{
271
+ source = "zenml-io/zenml"
272
+ }}
273
+ }}
274
+ }}
275
+
276
+ provider "azurerm" {{
277
+ features {{
278
+ resource_group {{
279
+ prevent_deletion_if_contains_resources = false
280
+ }}
281
+ }}
282
+ }}
283
+
284
+ provider "zenml" {{
285
+ server_url = "{self.zenml_server_url}"
286
+ api_token = "{self.zenml_server_api_token}"
287
+ }}
288
+
289
+ module "zenml_stack" {{
263
290
  source = "zenml-io/zenml-stack/azure"
264
291
 
265
292
  location = "{self.location or "eastus"}"
266
- orchestrator = "azureml"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
293
  zenml_stack_name = "{self.stack_name}"
271
294
  zenml_stack_deployment = "{self.deployment_type}"
272
295
  }}
@@ -70,11 +70,12 @@ and are aware of any potential costs:
70
70
 
71
71
  - A GCS bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/gcp).
72
72
  - A Google Artifact Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/gcp).
73
- - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex).
73
+ - Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
74
+ and as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/vertex).
74
75
  - GCP Cloud Build registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builders/gcp).
75
76
  - A GCP Service Account with the minimum necessary permissions to access the
76
77
  above resources.
77
- - An GCP Service Account access key used to give access to ZenML to connect to
78
+ - A GCP Service Account access key used to give access to ZenML to connect to
78
79
  the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/gcp-service-connector).
79
80
 
80
81
  The Deployment Manager deployment will automatically create a GCP Service
@@ -259,14 +260,30 @@ GCP project and to clean up the resources created by the stack by using
259
260
  )
260
261
 
261
262
  if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
262
- config = f"""module "zenml_stack" {{
263
+ config = f"""terraform {{
264
+ required_providers {{
265
+ google = {{
266
+ source = "hashicorp/google"
267
+ }}
268
+ zenml = {{
269
+ source = "zenml-io/zenml"
270
+ }}
271
+ }}
272
+ }}
273
+
274
+ provider "google" {{
275
+ region = "{self.location or "europe-west3"}"
276
+ project = your GCP project name
277
+ }}
278
+
279
+ provider "zenml" {{
280
+ server_url = "{self.zenml_server_url}"
281
+ api_token = "{self.zenml_server_api_token}"
282
+ }}
283
+
284
+ module "zenml_stack" {{
263
285
  source = "zenml-io/zenml-stack/gcp"
264
286
 
265
- project_id = "my-gcp-project"
266
- region = "{self.location or "europe-west3"}"
267
- zenml_server_url = "{self.zenml_server_url}"
268
- zenml_api_key = ""
269
- zenml_api_token = "{self.zenml_server_api_token}"
270
287
  zenml_stack_name = "{self.stack_name}"
271
288
  zenml_stack_deployment = "{self.deployment_type}"
272
289
  }}
@@ -219,16 +219,14 @@ class ZenMLCloudStackDeployment(BaseModel):
219
219
  if stack.labels.get("zenml:deployment") != self.deployment_type:
220
220
  continue
221
221
 
222
- artifact_store = stack.components[
223
- StackComponentType.ARTIFACT_STORE
224
- ][0]
222
+ orchestrator = stack.components[StackComponentType.ORCHESTRATOR][0]
225
223
 
226
- if not artifact_store.connector:
224
+ if not orchestrator.connector:
227
225
  continue
228
226
 
229
227
  return DeployedStack(
230
228
  stack=stack,
231
- service_connector=artifact_store.connector,
229
+ service_connector=orchestrator.connector,
232
230
  )
233
231
 
234
232
  return None
zenml/steps/base_step.py CHANGED
@@ -327,12 +327,12 @@ class BaseStep:
327
327
  The artifacts, external artifacts, model version artifacts/metadata and parameters for the step.
328
328
  """
329
329
  from zenml.artifacts.external_artifact import ExternalArtifact
330
+ from zenml.metadata.lazy_load import LazyRunMetadataResponse
330
331
  from zenml.model.lazy_load import ModelVersionDataLazyLoader
331
332
  from zenml.models.v2.core.artifact_version import (
332
333
  ArtifactVersionResponse,
333
334
  LazyArtifactVersionResponse,
334
335
  )
335
- from zenml.models.v2.core.run_metadata import LazyRunMetadataResponse
336
336
 
337
337
  signature = inspect.signature(self.entrypoint, follow_wrapped=True)
338
338
 
@@ -32,6 +32,7 @@ from zenml.constants import ENFORCE_TYPE_ANNOTATIONS
32
32
  from zenml.exceptions import StepInterfaceError
33
33
  from zenml.logger import get_logger
34
34
  from zenml.materializers.base_materializer import BaseMaterializer
35
+ from zenml.metadata.lazy_load import LazyRunMetadataResponse
35
36
  from zenml.steps.utils import (
36
37
  OutputSignature,
37
38
  parse_return_type_annotations,
@@ -136,10 +137,7 @@ class EntrypointFunctionDefinition(NamedTuple):
136
137
  UnmaterializedArtifact,
137
138
  )
138
139
  from zenml.client_lazy_loader import ClientLazyLoader
139
- from zenml.models import (
140
- ArtifactVersionResponse,
141
- RunMetadataResponse,
142
- )
140
+ from zenml.models import ArtifactVersionResponse
143
141
 
144
142
  if key not in self.inputs:
145
143
  raise KeyError(
@@ -154,8 +152,8 @@ class EntrypointFunctionDefinition(NamedTuple):
154
152
  StepArtifact,
155
153
  ExternalArtifact,
156
154
  ArtifactVersionResponse,
157
- RunMetadataResponse,
158
155
  ClientLazyLoader,
156
+ LazyRunMetadataResponse,
159
157
  ),
160
158
  ):
161
159
  # If we were to do any type validation for artifacts here, we
@@ -35,11 +35,12 @@ if TYPE_CHECKING:
35
35
  from zenml.metadata.metadata_types import MetadataType
36
36
  from zenml.model.model import Model
37
37
  from zenml.models import (
38
- ArtifactVersionResponse,
39
38
  PipelineResponse,
40
39
  PipelineRunResponse,
41
40
  StepRunResponse,
42
41
  )
42
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
+
43
44
 
44
45
  logger = get_logger(__name__)
45
46
 
@@ -191,7 +192,7 @@ class StepContext(metaclass=SingletonMetaClass):
191
192
  return self.model_version.to_model_class()
192
193
 
193
194
  @property
194
- def inputs(self) -> Dict[str, "ArtifactVersionResponse"]:
195
+ def inputs(self) -> Dict[str, "StepRunInputResponse"]:
195
196
  """Returns the input artifacts of the current step.
196
197
 
197
198
  Returns:
zenml/steps/utils.py CHANGED
@@ -26,7 +26,11 @@ from typing_extensions import Annotated
26
26
 
27
27
  from zenml.artifacts.artifact_config import ArtifactConfig
28
28
  from zenml.client import Client
29
- from zenml.enums import ExecutionStatus, MetadataResourceTypes
29
+ from zenml.enums import (
30
+ ArtifactSaveType,
31
+ ExecutionStatus,
32
+ MetadataResourceTypes,
33
+ )
30
34
  from zenml.exceptions import StepInterfaceError
31
35
  from zenml.logger import get_logger
32
36
  from zenml.metadata.metadata_types import MetadataType
@@ -438,6 +442,11 @@ def log_step_metadata(
438
442
  from within a step or if no pipeline name or ID is provided and
439
443
  the function is not called from within a step.
440
444
  """
445
+ logger.warning(
446
+ "The `log_step_metadata` function is deprecated and will soon be "
447
+ "removed. Please use `log_metadata` instead."
448
+ )
449
+
441
450
  step_context = None
442
451
  if not step_name:
443
452
  with contextlib.suppress(RuntimeError):
@@ -547,8 +556,10 @@ def run_as_single_step_pipeline(
547
556
  # 4. Load output artifacts
548
557
  step_run = next(iter(run.steps.values()))
549
558
  outputs = [
550
- step_run.outputs[output_name].load()
559
+ artifact_version.load()
551
560
  for output_name in step_run.config.outputs.keys()
561
+ for artifact_version in step_run.outputs[output_name]
562
+ if artifact_version.save_type == ArtifactSaveType.STEP_OUTPUT
552
563
  ]
553
564
 
554
565
  if len(outputs) == 0: