zenml-nightly 0.68.1.dev20241106__py3-none-any.whl → 0.70.0.dev20241116__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +77 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/artifacts/external_artifact.py +2 -1
  6. zenml/artifacts/utils.py +138 -79
  7. zenml/cli/base.py +4 -4
  8. zenml/cli/model.py +1 -6
  9. zenml/cli/stack.py +1 -0
  10. zenml/client.py +29 -74
  11. zenml/config/server_config.py +17 -1
  12. zenml/constants.py +2 -7
  13. zenml/data_validators/base_data_validator.py +2 -2
  14. zenml/enums.py +20 -4
  15. zenml/exceptions.py +4 -0
  16. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
  17. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  18. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
  19. zenml/integrations/evidently/__init__.py +1 -1
  20. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  21. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
  22. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
  23. zenml/integrations/vllm/services/vllm_deployment.py +16 -7
  24. zenml/metadata/lazy_load.py +20 -7
  25. zenml/model/model.py +17 -64
  26. zenml/model/utils.py +5 -0
  27. zenml/models/__init__.py +0 -12
  28. zenml/models/v2/base/filter.py +121 -8
  29. zenml/models/v2/core/artifact_version.py +42 -7
  30. zenml/models/v2/core/model_version.py +26 -5
  31. zenml/models/v2/core/pipeline_run.py +25 -6
  32. zenml/models/v2/core/run_metadata.py +2 -217
  33. zenml/models/v2/core/step_run.py +62 -24
  34. zenml/orchestrators/base_orchestrator.py +12 -1
  35. zenml/orchestrators/input_utils.py +44 -19
  36. zenml/orchestrators/step_launcher.py +4 -3
  37. zenml/orchestrators/step_run_utils.py +19 -15
  38. zenml/orchestrators/step_runner.py +21 -13
  39. zenml/orchestrators/utils.py +45 -26
  40. zenml/stack_deployments/aws_stack_deployment.py +23 -6
  41. zenml/stack_deployments/azure_stack_deployment.py +28 -5
  42. zenml/stack_deployments/gcp_stack_deployment.py +25 -8
  43. zenml/stack_deployments/stack_deployment.py +3 -5
  44. zenml/steps/base_step.py +1 -1
  45. zenml/steps/entrypoint_function_utils.py +3 -5
  46. zenml/steps/step_context.py +3 -2
  47. zenml/steps/utils.py +13 -2
  48. zenml/utils/metadata_utils.py +335 -0
  49. zenml/zen_server/auth.py +221 -3
  50. zenml/zen_server/cache.py +208 -0
  51. zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
  52. zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
  53. zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
  54. zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
  55. zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
  56. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
  57. zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
  58. zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
  59. zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
  60. zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
  61. zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
  62. zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
  63. zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
  64. zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
  65. zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
  66. zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
  67. zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
  68. zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
  69. zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
  71. zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
  72. zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
  73. zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
  74. zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
  75. zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
  76. zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
  77. zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
  78. zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
  79. zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
  80. zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
  81. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
  82. zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
  83. zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
  84. zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
  86. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
  87. zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
  88. zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
  89. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
  90. zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
  92. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
  93. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  94. zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
  95. zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
  96. zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
  99. zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
  100. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
  101. zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
  105. zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
  106. zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
  107. zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
  108. zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
  109. zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
  110. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
  111. zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
  112. zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
  113. zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
  114. zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
  115. zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
  116. zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
  117. zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
  118. zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
  119. zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
  120. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
  121. zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
  122. zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
  123. zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
  124. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
  125. zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
  126. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
  127. zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
  128. zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
  129. zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
  130. zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
  133. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
  134. zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
  135. zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
  136. zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
  137. zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
  138. zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
  139. zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
  142. zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
  143. zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
  144. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
  145. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
  146. zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
  147. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
  148. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
  149. zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
  150. zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
  151. zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
  152. zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
  153. zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
  155. zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
  156. zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
  157. zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
  158. zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
  159. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
  160. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  161. zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
  162. zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
  163. zenml/zen_server/dashboard/index.html +7 -7
  164. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  165. zenml/zen_server/deploy/helm/README.md +2 -2
  166. zenml/zen_server/exceptions.py +2 -0
  167. zenml/zen_server/jwt.py +30 -13
  168. zenml/zen_server/rbac/endpoint_utils.py +43 -1
  169. zenml/zen_server/rbac/utils.py +0 -2
  170. zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
  171. zenml/zen_server/routers/auth_endpoints.py +134 -102
  172. zenml/zen_server/routers/logs_endpoints.py +66 -0
  173. zenml/zen_server/routers/workspaces_endpoints.py +3 -4
  174. zenml/zen_server/template_execution/utils.py +14 -16
  175. zenml/zen_server/utils.py +27 -0
  176. zenml/zen_server/zen_server_api.py +6 -3
  177. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  178. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  179. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  180. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  181. zenml/zen_stores/rest_zen_store.py +68 -64
  182. zenml/zen_stores/schemas/artifact_schemas.py +8 -1
  183. zenml/zen_stores/schemas/model_schemas.py +27 -3
  184. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
  185. zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
  186. zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
  187. zenml/zen_stores/schemas/step_run_schemas.py +18 -10
  188. zenml/zen_stores/sql_zen_store.py +283 -219
  189. zenml/zen_stores/zen_store_interface.py +15 -42
  190. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/METADATA +2 -2
  191. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/RECORD +194 -179
  192. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
  193. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
  194. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
  195. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
  196. zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
  197. zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
  198. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
  199. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
  200. zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
  201. zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
  202. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  203. zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
  204. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
  205. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
  206. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
  207. zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
  208. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
  209. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
  210. zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
  211. zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
  212. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
  213. zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
  214. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
  215. zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
  216. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
  217. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
  218. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
  219. zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
  220. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
  221. zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
  222. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
  223. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
  224. zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
  225. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
  226. zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
  227. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
  228. zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
  229. zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
  230. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/LICENSE +0 -0
  231. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/WHEEL +0 -0
  232. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/entry_points.txt +0 -0
@@ -21,7 +21,8 @@ from pydantic import BaseModel, ConfigDict, Field
21
21
 
22
22
  from zenml.config.step_configurations import StepConfiguration, StepSpec
23
23
  from zenml.constants import STR_FIELD_MAX_LENGTH, TEXT_FIELD_MAX_LENGTH
24
- from zenml.enums import ExecutionStatus
24
+ from zenml.enums import ExecutionStatus, StepRunInputArtifactType
25
+ from zenml.metadata.metadata_types import MetadataType
25
26
  from zenml.models.v2.base.scoped import (
26
27
  WorkspaceScopedFilter,
27
28
  WorkspaceScopedRequest,
@@ -30,19 +31,35 @@ from zenml.models.v2.base.scoped import (
30
31
  WorkspaceScopedResponseMetadata,
31
32
  WorkspaceScopedResponseResources,
32
33
  )
34
+ from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
33
35
  from zenml.models.v2.core.model_version import ModelVersionResponse
34
36
 
35
37
  if TYPE_CHECKING:
36
38
  from sqlalchemy.sql.elements import ColumnElement
37
39
 
38
- from zenml.models.v2.core.artifact_version import ArtifactVersionResponse
39
40
  from zenml.models.v2.core.logs import (
40
41
  LogsRequest,
41
42
  LogsResponse,
42
43
  )
43
- from zenml.models.v2.core.run_metadata import (
44
- RunMetadataResponse,
45
- )
44
+
45
+
46
+ class StepRunInputResponse(ArtifactVersionResponse):
47
+ """Response model for step run inputs."""
48
+
49
+ input_type: StepRunInputArtifactType
50
+
51
+ def get_hydrated_version(self) -> "StepRunInputResponse":
52
+ """Get the hydrated version of this step run input.
53
+
54
+ Returns:
55
+ an instance of the same entity with the metadata field attached.
56
+ """
57
+ from zenml.client import Client
58
+
59
+ return StepRunInputResponse(
60
+ input_type=self.input_type,
61
+ **Client().zen_store.get_artifact_version(self.id).model_dump(),
62
+ )
46
63
 
47
64
 
48
65
  # ------------------ Request Model ------------------
@@ -97,11 +114,11 @@ class StepRunRequest(WorkspaceScopedRequest):
97
114
  )
98
115
  inputs: Dict[str, UUID] = Field(
99
116
  title="The IDs of the input artifact versions of the step run.",
100
- default={},
117
+ default_factory=dict,
101
118
  )
102
- outputs: Dict[str, UUID] = Field(
119
+ outputs: Dict[str, List[UUID]] = Field(
103
120
  title="The IDs of the output artifact versions of the step run.",
104
- default={},
121
+ default_factory=dict,
105
122
  )
106
123
  logs: Optional["LogsRequest"] = Field(
107
124
  title="Logs associated with this step run.",
@@ -129,10 +146,6 @@ class StepRunUpdate(BaseModel):
129
146
  title="The IDs of the output artifact versions of the step run.",
130
147
  default={},
131
148
  )
132
- saved_artifact_versions: Dict[str, UUID] = Field(
133
- title="The IDs of artifact versions that were saved by this step run.",
134
- default={},
135
- )
136
149
  loaded_artifact_versions: Dict[str, UUID] = Field(
137
150
  title="The IDs of artifact versions that were loaded by this step run.",
138
151
  default={},
@@ -166,13 +179,13 @@ class StepRunResponseBody(WorkspaceScopedResponseBody):
166
179
  title="The end time of the step run.",
167
180
  default=None,
168
181
  )
169
- inputs: Dict[str, "ArtifactVersionResponse"] = Field(
182
+ inputs: Dict[str, StepRunInputResponse] = Field(
170
183
  title="The input artifact versions of the step run.",
171
- default={},
184
+ default_factory=dict,
172
185
  )
173
- outputs: Dict[str, "ArtifactVersionResponse"] = Field(
186
+ outputs: Dict[str, List[ArtifactVersionResponse]] = Field(
174
187
  title="The output artifact versions of the step run.",
175
- default={},
188
+ default_factory=dict,
176
189
  )
177
190
  model_version_id: Optional[UUID] = Field(
178
191
  title="The ID of the model version that was "
@@ -230,7 +243,7 @@ class StepRunResponseMetadata(WorkspaceScopedResponseMetadata):
230
243
  title="The IDs of the parent steps of this step run.",
231
244
  default_factory=list,
232
245
  )
233
- run_metadata: Dict[str, "RunMetadataResponse"] = Field(
246
+ run_metadata: Dict[str, MetadataType] = Field(
234
247
  title="Metadata associated with this step run.",
235
248
  default={},
236
249
  )
@@ -274,7 +287,7 @@ class StepRunResponse(
274
287
 
275
288
  # Helper properties
276
289
  @property
277
- def input(self) -> "ArtifactVersionResponse":
290
+ def input(self) -> ArtifactVersionResponse:
278
291
  """Returns the input artifact that was used to run this step.
279
292
 
280
293
  Returns:
@@ -293,7 +306,7 @@ class StepRunResponse(
293
306
  return next(iter(self.inputs.values()))
294
307
 
295
308
  @property
296
- def output(self) -> "ArtifactVersionResponse":
309
+ def output(self) -> ArtifactVersionResponse:
297
310
  """Returns the output artifact that was written by this step.
298
311
 
299
312
  Returns:
@@ -304,12 +317,15 @@ class StepRunResponse(
304
317
  """
305
318
  if not self.outputs:
306
319
  raise ValueError(f"Step {self.name} has no outputs.")
307
- if len(self.outputs) > 1:
320
+ if len(self.outputs) > 1 or (
321
+ len(self.outputs) == 1
322
+ and len(next(iter(self.outputs.values()))) > 1
323
+ ):
308
324
  raise ValueError(
309
325
  f"Step {self.name} has multiple outputs, so `Step.output` is "
310
326
  "ambiguous. Please use `Step.outputs` instead."
311
327
  )
312
- return next(iter(self.outputs.values()))
328
+ return next(iter(self.outputs.values()))[0]
313
329
 
314
330
  # Body and metadata properties
315
331
  @property
@@ -322,7 +338,7 @@ class StepRunResponse(
322
338
  return self.get_body().status
323
339
 
324
340
  @property
325
- def inputs(self) -> Dict[str, "ArtifactVersionResponse"]:
341
+ def inputs(self) -> Dict[str, StepRunInputResponse]:
326
342
  """The `inputs` property.
327
343
 
328
344
  Returns:
@@ -331,7 +347,7 @@ class StepRunResponse(
331
347
  return self.get_body().inputs
332
348
 
333
349
  @property
334
- def outputs(self) -> Dict[str, "ArtifactVersionResponse"]:
350
+ def outputs(self) -> Dict[str, List[ArtifactVersionResponse]]:
335
351
  """The `outputs` property.
336
352
 
337
353
  Returns:
@@ -466,7 +482,7 @@ class StepRunResponse(
466
482
  return self.get_metadata().parent_step_ids
467
483
 
468
484
  @property
469
- def run_metadata(self) -> Dict[str, "RunMetadataResponse"]:
485
+ def run_metadata(self) -> Dict[str, MetadataType]:
470
486
  """The `run_metadata` property.
471
487
 
472
488
  Returns:
@@ -493,6 +509,7 @@ class StepRunFilter(WorkspaceScopedFilter):
493
509
  FILTER_EXCLUDE_FIELDS: ClassVar[List[str]] = [
494
510
  *WorkspaceScopedFilter.FILTER_EXCLUDE_FIELDS,
495
511
  "model",
512
+ "run_metadata",
496
513
  ]
497
514
 
498
515
  name: Optional[str] = Field(
@@ -555,6 +572,10 @@ class StepRunFilter(WorkspaceScopedFilter):
555
572
  default=None,
556
573
  description="Name/ID of the model associated with the step run.",
557
574
  )
575
+ run_metadata: Optional[Dict[str, str]] = Field(
576
+ default=None,
577
+ description="The run_metadata to filter the step runs by.",
578
+ )
558
579
 
559
580
  model_config = ConfigDict(protected_namespaces=())
560
581
 
@@ -573,6 +594,7 @@ class StepRunFilter(WorkspaceScopedFilter):
573
594
  from zenml.zen_stores.schemas import (
574
595
  ModelSchema,
575
596
  ModelVersionSchema,
597
+ RunMetadataSchema,
576
598
  StepRunSchema,
577
599
  )
578
600
 
@@ -585,5 +607,21 @@ class StepRunFilter(WorkspaceScopedFilter):
585
607
  ),
586
608
  )
587
609
  custom_filters.append(model_filter)
610
+ if self.run_metadata is not None:
611
+ from zenml.enums import MetadataResourceTypes
612
+
613
+ for key, value in self.run_metadata.items():
614
+ additional_filter = and_(
615
+ RunMetadataSchema.resource_id == StepRunSchema.id,
616
+ RunMetadataSchema.resource_type
617
+ == MetadataResourceTypes.STEP_RUN,
618
+ RunMetadataSchema.key == key,
619
+ self.generate_custom_query_conditions_for_column(
620
+ value=value,
621
+ table=RunMetadataSchema,
622
+ column="value",
623
+ ),
624
+ )
625
+ custom_filters.append(additional_filter)
588
626
 
589
627
  return custom_filters
@@ -15,6 +15,7 @@
15
15
 
16
16
  from abc import ABC, abstractmethod
17
17
  from typing import TYPE_CHECKING, Any, Dict, Iterator, Optional, Type, cast
18
+ from uuid import UUID
18
19
 
19
20
  from pydantic import model_validator
20
21
 
@@ -186,7 +187,17 @@ class BaseOrchestrator(StackComponent, ABC):
186
187
  """
187
188
  self._prepare_run(deployment=deployment)
188
189
 
189
- environment = get_config_environment_vars(deployment=deployment)
190
+ pipeline_run_id: Optional[UUID] = None
191
+ schedule_id: Optional[UUID] = None
192
+ if deployment.schedule:
193
+ schedule_id = deployment.schedule.id
194
+ if placeholder_run:
195
+ pipeline_run_id = placeholder_run.id
196
+
197
+ environment = get_config_environment_vars(
198
+ schedule_id=schedule_id,
199
+ pipeline_run_id=pipeline_run_id,
200
+ )
190
201
 
191
202
  prevent_client_side_caching = handle_bool_env_var(
192
203
  ENV_ZENML_PREVENT_CLIENT_SIDE_CACHING, default=False
@@ -18,17 +18,19 @@ from uuid import UUID
18
18
 
19
19
  from zenml.client import Client
20
20
  from zenml.config.step_configurations import Step
21
+ from zenml.enums import ArtifactSaveType, StepRunInputArtifactType
21
22
  from zenml.exceptions import InputResolutionError
22
23
  from zenml.utils import pagination_utils
23
24
 
24
25
  if TYPE_CHECKING:
25
- from zenml.models import ArtifactVersionResponse, PipelineRunResponse
26
+ from zenml.models import PipelineRunResponse
27
+ from zenml.models.v2.core.step_run import StepRunInputResponse
26
28
 
27
29
 
28
30
  def resolve_step_inputs(
29
31
  step: "Step",
30
32
  pipeline_run: "PipelineRunResponse",
31
- ) -> Tuple[Dict[str, "ArtifactVersionResponse"], List[UUID]]:
33
+ ) -> Tuple[Dict[str, "StepRunInputResponse"], List[UUID]]:
32
34
  """Resolves inputs for the current step.
33
35
 
34
36
  Args:
@@ -45,7 +47,8 @@ def resolve_step_inputs(
45
47
  The IDs of the input artifact versions and the IDs of parent steps of
46
48
  the current step.
47
49
  """
48
- from zenml.models import ArtifactVersionResponse, RunMetadataResponse
50
+ from zenml.models import ArtifactVersionResponse
51
+ from zenml.models.v2.core.step_run import StepRunInputResponse
49
52
 
50
53
  current_run_steps = {
51
54
  run_step.name: run_step
@@ -54,7 +57,7 @@ def resolve_step_inputs(
54
57
  )
55
58
  }
56
59
 
57
- input_artifacts: Dict[str, "ArtifactVersionResponse"] = {}
60
+ input_artifacts: Dict[str, StepRunInputResponse] = {}
58
61
  for name, input_ in step.spec.inputs.items():
59
62
  try:
60
63
  step_run = current_run_steps[input_.step_name]
@@ -64,22 +67,44 @@ def resolve_step_inputs(
64
67
  )
65
68
 
66
69
  try:
67
- artifact = step_run.outputs[input_.output_name]
70
+ outputs = step_run.outputs[input_.output_name]
68
71
  except KeyError:
69
72
  raise InputResolutionError(
70
- f"No output `{input_.output_name}` found for step "
73
+ f"No step output `{input_.output_name}` found for step "
71
74
  f"`{input_.step_name}`."
72
75
  )
73
76
 
74
- input_artifacts[name] = artifact
77
+ step_outputs = [
78
+ output
79
+ for output in outputs
80
+ if output.save_type == ArtifactSaveType.STEP_OUTPUT
81
+ ]
82
+ if len(step_outputs) > 2:
83
+ # This should never happen, there can only be a single regular step
84
+ # output for a name
85
+ raise InputResolutionError(
86
+ f"Too many step outputs for output `{input_.output_name}` of "
87
+ f"step `{input_.step_name}`."
88
+ )
89
+ elif len(step_outputs) == 0:
90
+ raise InputResolutionError(
91
+ f"No step output `{input_.output_name}` found for step "
92
+ f"`{input_.step_name}`."
93
+ )
94
+
95
+ input_artifacts[name] = StepRunInputResponse(
96
+ input_type=StepRunInputArtifactType.STEP_OUTPUT,
97
+ **step_outputs[0].model_dump(),
98
+ )
75
99
 
76
100
  for (
77
101
  name,
78
102
  external_artifact,
79
103
  ) in step.config.external_input_artifacts.items():
80
104
  artifact_version_id = external_artifact.get_artifact_version_id()
81
- input_artifacts[name] = Client().get_artifact_version(
82
- artifact_version_id
105
+ input_artifacts[name] = StepRunInputResponse(
106
+ input_type=StepRunInputArtifactType.EXTERNAL,
107
+ **Client().get_artifact_version(artifact_version_id).model_dump(),
83
108
  )
84
109
 
85
110
  for name, config_ in step.config.model_artifacts_or_metadata.items():
@@ -98,9 +123,7 @@ def resolve_step_inputs(
98
123
  ):
99
124
  # metadata values should go directly in parameters, as primitive types
100
125
  step.config.parameters[name] = (
101
- context_model_version.run_metadata[
102
- config_.metadata_name
103
- ].value
126
+ context_model_version.run_metadata[config_.metadata_name]
104
127
  )
105
128
  elif config_.artifact_name is None:
106
129
  err_msg = (
@@ -112,14 +135,15 @@ def resolve_step_inputs(
112
135
  config_.artifact_name, config_.artifact_version
113
136
  ):
114
137
  if config_.metadata_name is None:
115
- input_artifacts[name] = artifact_
138
+ input_artifacts[name] = StepRunInputResponse(
139
+ input_type=StepRunInputArtifactType.LAZY_LOADED,
140
+ **artifact_.model_dump(),
141
+ )
116
142
  elif config_.metadata_name:
117
143
  # metadata values should go directly in parameters, as primitive types
118
144
  try:
119
145
  step.config.parameters[name] = (
120
- artifact_.run_metadata[
121
- config_.metadata_name
122
- ].value
146
+ artifact_.run_metadata[config_.metadata_name]
123
147
  )
124
148
  except KeyError:
125
149
  err_msg = (
@@ -141,9 +165,10 @@ def resolve_step_inputs(
141
165
  for name, cll_ in step.config.client_lazy_loaders.items():
142
166
  value_ = cll_.evaluate()
143
167
  if isinstance(value_, ArtifactVersionResponse):
144
- input_artifacts[name] = value_
145
- elif isinstance(value_, RunMetadataResponse):
146
- step.config.parameters[name] = value_.value
168
+ input_artifacts[name] = StepRunInputResponse(
169
+ input_type=StepRunInputArtifactType.LAZY_LOADED,
170
+ **value_.model_dump(),
171
+ )
147
172
  else:
148
173
  step.config.parameters[name] = value_
149
174
 
@@ -33,13 +33,13 @@ from zenml.environment import get_run_environment_dict
33
33
  from zenml.logger import get_logger
34
34
  from zenml.logging import step_logging
35
35
  from zenml.models import (
36
- ArtifactVersionResponse,
37
36
  LogsRequest,
38
37
  PipelineDeploymentResponse,
39
38
  PipelineRunRequest,
40
39
  PipelineRunResponse,
41
40
  StepRunResponse,
42
41
  )
42
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
43
  from zenml.orchestrators import output_utils, publish_utils, step_run_utils
44
44
  from zenml.orchestrators import utils as orchestrator_utils
45
45
  from zenml.orchestrators.step_runner import StepRunner
@@ -422,7 +422,8 @@ class StepLauncher:
422
422
  )
423
423
  )
424
424
  environment = orchestrator_utils.get_config_environment_vars(
425
- deployment=self._deployment
425
+ pipeline_run_id=step_run_info.run_id,
426
+ step_run_id=step_run_info.step_run_id,
426
427
  )
427
428
  if last_retry:
428
429
  environment[ENV_ZENML_IGNORE_FAILURE_HOOK] = str(False)
@@ -442,7 +443,7 @@ class StepLauncher:
442
443
  pipeline_run: PipelineRunResponse,
443
444
  step_run: StepRunResponse,
444
445
  step_run_info: StepRunInfo,
445
- input_artifacts: Dict[str, ArtifactVersionResponse],
446
+ input_artifacts: Dict[str, StepRunInputResponse],
446
447
  output_artifact_uris: Dict[str, str],
447
448
  last_retry: bool,
448
449
  ) -> None:
@@ -14,12 +14,12 @@
14
14
  """Utilities for creating step runs."""
15
15
 
16
16
  from datetime import datetime
17
- from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
17
+ from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple
18
18
 
19
19
  from zenml.client import Client
20
20
  from zenml.config.step_configurations import ArtifactConfiguration, Step
21
21
  from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
22
- from zenml.enums import ExecutionStatus
22
+ from zenml.enums import ArtifactSaveType, ExecutionStatus
23
23
  from zenml.logger import get_logger
24
24
  from zenml.model.utils import link_artifact_version_to_model_version
25
25
  from zenml.models import (
@@ -104,6 +104,7 @@ class StepRunRequestFactory:
104
104
  input_name: artifact.id
105
105
  for input_name, artifact in input_artifacts.items()
106
106
  }
107
+
107
108
  request.inputs = input_artifact_ids
108
109
  request.parent_step_ids = parent_step_ids
109
110
 
@@ -142,8 +143,8 @@ class StepRunRequestFactory:
142
143
 
143
144
  request.original_step_run_id = cached_step_run.id
144
145
  request.outputs = {
145
- output_name: artifact.id
146
- for output_name, artifact in cached_step_run.outputs.items()
146
+ output_name: [artifact.id for artifact in artifacts]
147
+ for output_name, artifacts in cached_step_run.outputs.items()
147
148
  }
148
149
 
149
150
  request.status = ExecutionStatus.CACHED
@@ -551,7 +552,7 @@ def link_pipeline_run_to_model_version(
551
552
 
552
553
 
553
554
  def link_output_artifacts_to_model_version(
554
- artifacts: Dict[str, ArtifactVersionResponse],
555
+ artifacts: Dict[str, List[ArtifactVersionResponse]],
555
556
  output_configurations: Mapping[str, ArtifactConfiguration],
556
557
  model_version: ModelVersionResponse,
557
558
  ) -> None:
@@ -562,13 +563,16 @@ def link_output_artifacts_to_model_version(
562
563
  output_configurations: The output configurations for the step.
563
564
  model_version: The model version to link.
564
565
  """
565
- for output_name, output_artifact in artifacts.items():
566
- artifact_config = None
567
- if output_config := output_configurations.get(output_name, None):
568
- artifact_config = output_config.artifact_config
569
-
570
- link_artifact_version_to_model_version(
571
- artifact_version=output_artifact,
572
- model_version=model_version,
573
- artifact_config=artifact_config,
574
- )
566
+ for output_name, output_artifacts in artifacts.items():
567
+ for output_artifact in output_artifacts:
568
+ artifact_config = None
569
+ if output_artifact.save_type == ArtifactSaveType.STEP_OUTPUT and (
570
+ output_config := output_configurations.get(output_name, None)
571
+ ):
572
+ artifact_config = output_config.artifact_config
573
+
574
+ link_artifact_version_to_model_version(
575
+ artifact_version=output_artifact,
576
+ model_version=model_version,
577
+ artifact_config=artifact_config,
578
+ )
@@ -28,7 +28,8 @@ from typing import (
28
28
  )
29
29
 
30
30
  from zenml.artifacts.unmaterialized_artifact import UnmaterializedArtifact
31
- from zenml.artifacts.utils import save_artifact
31
+ from zenml.artifacts.utils import _store_artifact_data_and_prepare_request
32
+ from zenml.client import Client
32
33
  from zenml.config.step_configurations import StepConfiguration
33
34
  from zenml.config.step_run_info import StepRunInfo
34
35
  from zenml.constants import (
@@ -36,10 +37,12 @@ from zenml.constants import (
36
37
  ENV_ZENML_IGNORE_FAILURE_HOOK,
37
38
  handle_bool_env_var,
38
39
  )
40
+ from zenml.enums import ArtifactSaveType
39
41
  from zenml.exceptions import StepInterfaceError
40
42
  from zenml.logger import get_logger
41
43
  from zenml.logging.step_logging import StepLogsStorageContext, redirected
42
44
  from zenml.materializers.base_materializer import BaseMaterializer
45
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
46
  from zenml.orchestrators.publish_utils import (
44
47
  publish_step_run_metadata,
45
48
  publish_successful_step_run,
@@ -98,7 +101,7 @@ class StepRunner:
98
101
  self,
99
102
  pipeline_run: "PipelineRunResponse",
100
103
  step_run: "StepRunResponse",
101
- input_artifacts: Dict[str, "ArtifactVersionResponse"],
104
+ input_artifacts: Dict[str, StepRunInputResponse],
102
105
  output_artifact_uris: Dict[str, str],
103
106
  step_run_info: StepRunInfo,
104
107
  ) -> None:
@@ -241,7 +244,9 @@ class StepRunner:
241
244
  from zenml.orchestrators import step_run_utils
242
245
 
243
246
  step_run_utils.link_output_artifacts_to_model_version(
244
- artifacts=output_artifacts,
247
+ artifacts={
248
+ k: [v] for k, v in output_artifacts.items()
249
+ },
245
250
  output_configurations=step_run.config.outputs,
246
251
  model_version=model_version,
247
252
  )
@@ -302,7 +307,7 @@ class StepRunner:
302
307
  self,
303
308
  args: List[str],
304
309
  annotations: Dict[str, Any],
305
- input_artifacts: Dict[str, "ArtifactVersionResponse"],
310
+ input_artifacts: Dict[str, StepRunInputResponse],
306
311
  ) -> Dict[str, Any]:
307
312
  """Parses the inputs for a step entrypoint function.
308
313
 
@@ -534,7 +539,7 @@ class StepRunner:
534
539
  The IDs of the published output artifacts.
535
540
  """
536
541
  step_context = get_step_context()
537
- output_artifacts: Dict[str, "ArtifactVersionResponse"] = {}
542
+ artifact_requests = []
538
543
 
539
544
  for output_name, return_value in output_data.items():
540
545
  data_type = type(return_value)
@@ -595,22 +600,25 @@ class StepRunner:
595
600
  # Get full set of tags
596
601
  tags = step_context.get_output_tags(output_name)
597
602
 
598
- artifact = save_artifact(
603
+ artifact_request = _store_artifact_data_and_prepare_request(
599
604
  name=artifact_name,
600
605
  data=return_value,
601
- materializer=materializer_class,
606
+ materializer_class=materializer_class,
602
607
  uri=uri,
603
- extract_metadata=artifact_metadata_enabled,
604
- include_visualizations=artifact_visualization_enabled,
608
+ store_metadata=artifact_metadata_enabled,
609
+ store_visualizations=artifact_visualization_enabled,
605
610
  has_custom_name=has_custom_name,
606
611
  version=version,
607
612
  tags=tags,
608
- user_metadata=user_metadata,
609
- manual_save=False,
613
+ save_type=ArtifactSaveType.STEP_OUTPUT,
614
+ metadata=user_metadata,
610
615
  )
611
- output_artifacts[output_name] = artifact
616
+ artifact_requests.append(artifact_request)
612
617
 
613
- return output_artifacts
618
+ responses = Client().zen_store.batch_create_artifact_versions(
619
+ artifact_requests
620
+ )
621
+ return dict(zip(output_data.keys(), responses))
614
622
 
615
623
  def load_and_run_hook(
616
624
  self,