zenml-nightly 0.68.1.dev20241103__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +77 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/artifacts/external_artifact.py +2 -1
  6. zenml/artifacts/{load_directory_materializer.py → preexisting_data_materializer.py} +8 -9
  7. zenml/artifacts/utils.py +139 -80
  8. zenml/cli/base.py +4 -4
  9. zenml/cli/model.py +1 -6
  10. zenml/cli/stack.py +1 -0
  11. zenml/client.py +29 -74
  12. zenml/config/server_config.py +17 -1
  13. zenml/constants.py +2 -7
  14. zenml/data_validators/base_data_validator.py +2 -2
  15. zenml/enums.py +20 -4
  16. zenml/exceptions.py +4 -0
  17. zenml/integrations/__init__.py +3 -1
  18. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
  19. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  20. zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
  21. zenml/integrations/constants.py +1 -0
  22. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
  23. zenml/integrations/evidently/__init__.py +1 -1
  24. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  25. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +8 -12
  26. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +17 -18
  27. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +2 -5
  28. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +17 -18
  29. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -3
  30. zenml/integrations/langchain/__init__.py +2 -1
  31. zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
  32. zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
  33. zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
  34. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
  35. zenml/integrations/openai/__init__.py +1 -1
  36. zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
  37. zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
  38. zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
  39. zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
  40. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
  41. zenml/integrations/tensorflow/materializers/keras_materializer.py +11 -22
  42. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +8 -15
  43. zenml/integrations/vllm/__init__.py +50 -0
  44. zenml/integrations/vllm/flavors/__init__.py +21 -0
  45. zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
  46. zenml/integrations/vllm/model_deployers/__init__.py +19 -0
  47. zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
  48. zenml/integrations/vllm/services/__init__.py +19 -0
  49. zenml/integrations/vllm/services/vllm_deployment.py +206 -0
  50. zenml/integrations/whylogs/materializers/whylogs_materializer.py +11 -18
  51. zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
  52. zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
  53. zenml/materializers/base_materializer.py +68 -1
  54. zenml/metadata/lazy_load.py +20 -7
  55. zenml/model/model.py +17 -64
  56. zenml/model/utils.py +5 -0
  57. zenml/models/__init__.py +0 -12
  58. zenml/models/v2/base/filter.py +121 -8
  59. zenml/models/v2/core/artifact_version.py +42 -7
  60. zenml/models/v2/core/model_version.py +26 -5
  61. zenml/models/v2/core/pipeline_run.py +25 -6
  62. zenml/models/v2/core/run_metadata.py +2 -217
  63. zenml/models/v2/core/step_run.py +62 -24
  64. zenml/orchestrators/base_orchestrator.py +12 -1
  65. zenml/orchestrators/input_utils.py +44 -19
  66. zenml/orchestrators/step_launcher.py +4 -3
  67. zenml/orchestrators/step_run_utils.py +19 -15
  68. zenml/orchestrators/step_runner.py +25 -14
  69. zenml/orchestrators/utils.py +45 -26
  70. zenml/stack/flavor.py +9 -5
  71. zenml/stack_deployments/aws_stack_deployment.py +23 -6
  72. zenml/stack_deployments/azure_stack_deployment.py +28 -5
  73. zenml/stack_deployments/gcp_stack_deployment.py +25 -8
  74. zenml/stack_deployments/stack_deployment.py +3 -5
  75. zenml/steps/base_step.py +1 -1
  76. zenml/steps/entrypoint_function_utils.py +3 -5
  77. zenml/steps/step_context.py +5 -2
  78. zenml/steps/utils.py +13 -2
  79. zenml/utils/callback_registry.py +71 -0
  80. zenml/utils/metadata_utils.py +335 -0
  81. zenml/zen_server/auth.py +221 -3
  82. zenml/zen_server/cache.py +208 -0
  83. zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
  84. zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
  86. zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
  87. zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
  88. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
  89. zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
  90. zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
  92. zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
  93. zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
  94. zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
  95. zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
  96. zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
  99. zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
  100. zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
  101. zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
  105. zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
  106. zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
  107. zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
  108. zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
  109. zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
  110. zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
  111. zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
  112. zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
  113. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
  114. zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
  115. zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
  116. zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
  117. zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
  118. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
  119. zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
  120. zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
  121. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
  123. zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
  124. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
  125. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  126. zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
  127. zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
  128. zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
  129. zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
  130. zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
  131. zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
  132. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
  133. zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
  135. zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
  136. zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
  137. zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
  138. zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
  139. zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
  140. zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
  141. zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
  142. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
  143. zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
  144. zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
  145. zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
  147. zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
  148. zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
  150. zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
  151. zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
  152. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
  153. zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
  154. zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
  155. zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
  156. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
  157. zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
  158. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
  159. zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
  160. zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
  161. zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
  162. zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
  163. zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
  164. zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
  165. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
  166. zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
  167. zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
  168. zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
  169. zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
  170. zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
  171. zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
  172. zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
  173. zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
  174. zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
  175. zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
  176. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
  177. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
  178. zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
  179. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
  180. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
  181. zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
  182. zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
  183. zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
  184. zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
  185. zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
  186. zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
  187. zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
  188. zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
  189. zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
  190. zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
  191. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
  192. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  193. zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
  194. zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
  195. zenml/zen_server/dashboard/index.html +7 -7
  196. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  197. zenml/zen_server/deploy/helm/README.md +2 -2
  198. zenml/zen_server/exceptions.py +2 -0
  199. zenml/zen_server/jwt.py +30 -13
  200. zenml/zen_server/rbac/endpoint_utils.py +43 -1
  201. zenml/zen_server/rbac/utils.py +0 -2
  202. zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
  203. zenml/zen_server/routers/auth_endpoints.py +134 -102
  204. zenml/zen_server/routers/logs_endpoints.py +66 -0
  205. zenml/zen_server/routers/workspaces_endpoints.py +3 -4
  206. zenml/zen_server/template_execution/utils.py +14 -16
  207. zenml/zen_server/utils.py +27 -0
  208. zenml/zen_server/zen_server_api.py +6 -3
  209. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  210. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  211. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  212. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  213. zenml/zen_stores/rest_zen_store.py +68 -64
  214. zenml/zen_stores/schemas/artifact_schemas.py +8 -1
  215. zenml/zen_stores/schemas/model_schemas.py +27 -3
  216. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
  217. zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
  218. zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
  219. zenml/zen_stores/schemas/step_run_schemas.py +18 -10
  220. zenml/zen_stores/sql_zen_store.py +283 -219
  221. zenml/zen_stores/zen_store_interface.py +15 -42
  222. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
  223. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +226 -203
  224. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
  225. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
  226. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
  227. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
  228. zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
  229. zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
  230. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
  231. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
  232. zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
  233. zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
  234. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  235. zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
  236. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
  237. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
  238. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
  239. zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
  240. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
  241. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
  242. zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
  243. zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
  244. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
  245. zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
  246. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
  247. zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
  248. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
  249. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
  250. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
  251. zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
  252. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
  253. zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
  254. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
  255. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
  256. zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
  257. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
  258. zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
  259. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
  260. zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
  261. zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
  262. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
  263. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
  264. {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
zenml/steps/base_step.py CHANGED
@@ -327,12 +327,12 @@ class BaseStep:
327
327
  The artifacts, external artifacts, model version artifacts/metadata and parameters for the step.
328
328
  """
329
329
  from zenml.artifacts.external_artifact import ExternalArtifact
330
+ from zenml.metadata.lazy_load import LazyRunMetadataResponse
330
331
  from zenml.model.lazy_load import ModelVersionDataLazyLoader
331
332
  from zenml.models.v2.core.artifact_version import (
332
333
  ArtifactVersionResponse,
333
334
  LazyArtifactVersionResponse,
334
335
  )
335
- from zenml.models.v2.core.run_metadata import LazyRunMetadataResponse
336
336
 
337
337
  signature = inspect.signature(self.entrypoint, follow_wrapped=True)
338
338
 
@@ -32,6 +32,7 @@ from zenml.constants import ENFORCE_TYPE_ANNOTATIONS
32
32
  from zenml.exceptions import StepInterfaceError
33
33
  from zenml.logger import get_logger
34
34
  from zenml.materializers.base_materializer import BaseMaterializer
35
+ from zenml.metadata.lazy_load import LazyRunMetadataResponse
35
36
  from zenml.steps.utils import (
36
37
  OutputSignature,
37
38
  parse_return_type_annotations,
@@ -136,10 +137,7 @@ class EntrypointFunctionDefinition(NamedTuple):
136
137
  UnmaterializedArtifact,
137
138
  )
138
139
  from zenml.client_lazy_loader import ClientLazyLoader
139
- from zenml.models import (
140
- ArtifactVersionResponse,
141
- RunMetadataResponse,
142
- )
140
+ from zenml.models import ArtifactVersionResponse
143
141
 
144
142
  if key not in self.inputs:
145
143
  raise KeyError(
@@ -154,8 +152,8 @@ class EntrypointFunctionDefinition(NamedTuple):
154
152
  StepArtifact,
155
153
  ExternalArtifact,
156
154
  ArtifactVersionResponse,
157
- RunMetadataResponse,
158
155
  ClientLazyLoader,
156
+ LazyRunMetadataResponse,
159
157
  ),
160
158
  ):
161
159
  # If we were to do any type validation for artifacts here, we
@@ -26,6 +26,7 @@ from typing import (
26
26
 
27
27
  from zenml.exceptions import StepContextError
28
28
  from zenml.logger import get_logger
29
+ from zenml.utils.callback_registry import CallbackRegistry
29
30
  from zenml.utils.singleton import SingletonMetaClass
30
31
 
31
32
  if TYPE_CHECKING:
@@ -34,11 +35,12 @@ if TYPE_CHECKING:
34
35
  from zenml.metadata.metadata_types import MetadataType
35
36
  from zenml.model.model import Model
36
37
  from zenml.models import (
37
- ArtifactVersionResponse,
38
38
  PipelineResponse,
39
39
  PipelineRunResponse,
40
40
  StepRunResponse,
41
41
  )
42
+ from zenml.models.v2.core.step_run import StepRunInputResponse
43
+
42
44
 
43
45
  logger = get_logger(__name__)
44
46
 
@@ -145,6 +147,7 @@ class StepContext(metaclass=SingletonMetaClass):
145
147
  )
146
148
  for key in output_materializers.keys()
147
149
  }
150
+ self._cleanup_registry = CallbackRegistry()
148
151
 
149
152
  @property
150
153
  def pipeline(self) -> "PipelineResponse":
@@ -189,7 +192,7 @@ class StepContext(metaclass=SingletonMetaClass):
189
192
  return self.model_version.to_model_class()
190
193
 
191
194
  @property
192
- def inputs(self) -> Dict[str, "ArtifactVersionResponse"]:
195
+ def inputs(self) -> Dict[str, "StepRunInputResponse"]:
193
196
  """Returns the input artifacts of the current step.
194
197
 
195
198
  Returns:
zenml/steps/utils.py CHANGED
@@ -26,7 +26,11 @@ from typing_extensions import Annotated
26
26
 
27
27
  from zenml.artifacts.artifact_config import ArtifactConfig
28
28
  from zenml.client import Client
29
- from zenml.enums import ExecutionStatus, MetadataResourceTypes
29
+ from zenml.enums import (
30
+ ArtifactSaveType,
31
+ ExecutionStatus,
32
+ MetadataResourceTypes,
33
+ )
30
34
  from zenml.exceptions import StepInterfaceError
31
35
  from zenml.logger import get_logger
32
36
  from zenml.metadata.metadata_types import MetadataType
@@ -438,6 +442,11 @@ def log_step_metadata(
438
442
  from within a step or if no pipeline name or ID is provided and
439
443
  the function is not called from within a step.
440
444
  """
445
+ logger.warning(
446
+ "The `log_step_metadata` function is deprecated and will soon be "
447
+ "removed. Please use `log_metadata` instead."
448
+ )
449
+
441
450
  step_context = None
442
451
  if not step_name:
443
452
  with contextlib.suppress(RuntimeError):
@@ -547,8 +556,10 @@ def run_as_single_step_pipeline(
547
556
  # 4. Load output artifacts
548
557
  step_run = next(iter(run.steps.values()))
549
558
  outputs = [
550
- step_run.outputs[output_name].load()
559
+ artifact_version.load()
551
560
  for output_name in step_run.config.outputs.keys()
561
+ for artifact_version in step_run.outputs[output_name]
562
+ if artifact_version.save_type == ArtifactSaveType.STEP_OUTPUT
552
563
  ]
553
564
 
554
565
  if len(outputs) == 0:
@@ -0,0 +1,71 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Callback registry implementation."""
15
+
16
+ from typing import Any, Callable, Dict, List, Tuple
17
+
18
+ from typing_extensions import ParamSpec
19
+
20
+ from zenml.logger import get_logger
21
+
22
+ P = ParamSpec("P")
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ class CallbackRegistry:
28
+ """Callback registry class."""
29
+
30
+ def __init__(self) -> None:
31
+ """Initializes the callback registry."""
32
+ self._callbacks: List[
33
+ Tuple[Callable[P, Any], Tuple[Any], Dict[str, Any]]
34
+ ] = []
35
+
36
+ def register_callback(
37
+ self, callback: Callable[P, Any], *args: P.args, **kwargs: P.kwargs
38
+ ) -> None:
39
+ """Register a callback.
40
+
41
+ Args:
42
+ callback: The callback to register.
43
+ *args: Arguments to call the callback with.
44
+ **kwargs: Keyword arguments to call the callback with.
45
+ """
46
+ self._callbacks.append((callback, args, kwargs)) # type: ignore[arg-type]
47
+
48
+ def reset(self) -> None:
49
+ """Reset the callbacks."""
50
+ self._callbacks = []
51
+
52
+ def execute_callbacks(self, raise_on_exception: bool) -> None:
53
+ """Execute all registered callbacks.
54
+
55
+ Args:
56
+ raise_on_exception: If True, exceptions raised during the execution
57
+ of the callbacks will be raised. If False, a warning with the
58
+ exception will be logged instead.
59
+
60
+ Raises:
61
+ Exception: Exceptions raised in any of the callbacks if
62
+ `raise_on_exception` is set to True.
63
+ """
64
+ for callback, args, kwargs in self._callbacks:
65
+ try:
66
+ callback(*args, **kwargs)
67
+ except Exception as e:
68
+ if raise_on_exception:
69
+ raise e
70
+ else:
71
+ logger.warning("Failed to run callback: %s", str(e))
@@ -0,0 +1,335 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Utility functions to handle metadata for ZenML entities."""
15
+
16
+ import contextlib
17
+ from typing import Dict, Optional, Union, overload
18
+ from uuid import UUID
19
+
20
+ from zenml.client import Client
21
+ from zenml.enums import MetadataResourceTypes
22
+ from zenml.logger import get_logger
23
+ from zenml.metadata.metadata_types import MetadataType
24
+ from zenml.steps.step_context import get_step_context
25
+
26
+ logger = get_logger(__name__)
27
+
28
+
29
+ @overload
30
+ def log_metadata(metadata: Dict[str, MetadataType]) -> None: ...
31
+
32
+
33
+ @overload
34
+ def log_metadata(
35
+ *,
36
+ metadata: Dict[str, MetadataType],
37
+ artifact_version_id: UUID,
38
+ ) -> None: ...
39
+
40
+
41
+ @overload
42
+ def log_metadata(
43
+ *,
44
+ metadata: Dict[str, MetadataType],
45
+ artifact_name: str,
46
+ artifact_version: Optional[str] = None,
47
+ ) -> None: ...
48
+
49
+
50
+ @overload
51
+ def log_metadata(
52
+ *,
53
+ metadata: Dict[str, MetadataType],
54
+ model_version_id: UUID,
55
+ ) -> None: ...
56
+
57
+
58
+ @overload
59
+ def log_metadata(
60
+ *,
61
+ metadata: Dict[str, MetadataType],
62
+ model_name: str,
63
+ model_version: str,
64
+ ) -> None: ...
65
+
66
+
67
+ @overload
68
+ def log_metadata(
69
+ *,
70
+ metadata: Dict[str, MetadataType],
71
+ step_id: UUID,
72
+ ) -> None: ...
73
+
74
+
75
+ @overload
76
+ def log_metadata(
77
+ *,
78
+ metadata: Dict[str, MetadataType],
79
+ run_id_name_or_prefix: Union[UUID, str],
80
+ ) -> None: ...
81
+
82
+
83
+ @overload
84
+ def log_metadata(
85
+ *,
86
+ metadata: Dict[str, MetadataType],
87
+ step_name: str,
88
+ run_id_name_or_prefix: Union[UUID, str],
89
+ ) -> None: ...
90
+
91
+
92
+ def log_metadata(
93
+ metadata: Dict[str, MetadataType],
94
+ # Parameters to manually log metadata for steps and runs
95
+ step_id: Optional[UUID] = None,
96
+ step_name: Optional[str] = None,
97
+ run_id_name_or_prefix: Optional[Union[UUID, str]] = None,
98
+ # Parameters to manually log metadata for artifacts
99
+ artifact_version_id: Optional[UUID] = None,
100
+ artifact_name: Optional[str] = None,
101
+ artifact_version: Optional[str] = None,
102
+ # Parameters to manually log metadata for models
103
+ model_version_id: Optional[UUID] = None,
104
+ model_name: Optional[str] = None,
105
+ model_version: Optional[str] = None,
106
+ ) -> None:
107
+ """Logs metadata for various resource types in a generalized way.
108
+
109
+ Args:
110
+ metadata: The metadata to log.
111
+ step_id: The ID of the step.
112
+ step_name: The name of the step.
113
+ run_id_name_or_prefix: The id, name or prefix of the run
114
+ artifact_version_id: The ID of the artifact version
115
+ artifact_name: The name of the artifact.
116
+ artifact_version: The version of the artifact.
117
+ model_version_id: The ID of the model version.
118
+ model_name: The name of the model.
119
+ model_version: The version of the model
120
+
121
+ Raises:
122
+ ValueError: If no identifiers are provided and the function is not
123
+ called from within a step.
124
+ """
125
+ client = Client()
126
+
127
+ # If a step name is provided, we need a run_id_name_or_prefix and will log
128
+ # metadata for the steps pipeline and model accordingly.
129
+ if step_name is not None and run_id_name_or_prefix is not None:
130
+ run_model = client.get_pipeline_run(
131
+ name_id_or_prefix=run_id_name_or_prefix
132
+ )
133
+ step_model = run_model.steps[step_name]
134
+
135
+ client.create_run_metadata(
136
+ metadata=metadata,
137
+ resource_id=run_model.id,
138
+ resource_type=MetadataResourceTypes.PIPELINE_RUN,
139
+ )
140
+ client.create_run_metadata(
141
+ metadata=metadata,
142
+ resource_id=step_model.id,
143
+ resource_type=MetadataResourceTypes.STEP_RUN,
144
+ )
145
+ if step_model.model_version:
146
+ client.create_run_metadata(
147
+ metadata=metadata,
148
+ resource_id=step_model.model_version.id,
149
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
150
+ )
151
+
152
+ # If a step is identified by id, fetch it directly through the client,
153
+ # follow a similar procedure and log metadata for its pipeline and model
154
+ # as well.
155
+ elif step_id is not None:
156
+ step_model = client.get_run_step(step_run_id=step_id)
157
+ client.create_run_metadata(
158
+ metadata=metadata,
159
+ resource_id=step_model.pipeline_run_id,
160
+ resource_type=MetadataResourceTypes.PIPELINE_RUN,
161
+ )
162
+ client.create_run_metadata(
163
+ metadata=metadata,
164
+ resource_id=step_model.id,
165
+ resource_type=MetadataResourceTypes.STEP_RUN,
166
+ )
167
+ if step_model.model_version:
168
+ client.create_run_metadata(
169
+ metadata=metadata,
170
+ resource_id=step_model.model_version.id,
171
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
172
+ )
173
+
174
+ # If a pipeline run id is identified, we need to log metadata to it and its
175
+ # model as well.
176
+ elif run_id_name_or_prefix is not None:
177
+ run_model = client.get_pipeline_run(
178
+ name_id_or_prefix=run_id_name_or_prefix
179
+ )
180
+ client.create_run_metadata(
181
+ metadata=metadata,
182
+ resource_id=run_model.id,
183
+ resource_type=MetadataResourceTypes.PIPELINE_RUN,
184
+ )
185
+ if run_model.model_version:
186
+ client.create_run_metadata(
187
+ metadata=metadata,
188
+ resource_id=run_model.model_version.id,
189
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
190
+ )
191
+
192
+ # If the user provides a model name and version, we use to model abstraction
193
+ # to fetch the model version and attach the corresponding metadata to it.
194
+ elif model_name is not None and model_version is not None:
195
+ from zenml import Model
196
+
197
+ mv = Model(name=model_name, version=model_version)
198
+ client.create_run_metadata(
199
+ metadata=metadata,
200
+ resource_id=mv.id,
201
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
202
+ )
203
+
204
+ # If the user provides a model version id, we use the client to fetch it and
205
+ # attach the metadata to it.
206
+ elif model_version_id is not None:
207
+ model_version_id = client.get_model_version(
208
+ model_version_name_or_number_or_id=model_version_id
209
+ ).id
210
+ client.create_run_metadata(
211
+ metadata=metadata,
212
+ resource_id=model_version_id,
213
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
214
+ )
215
+
216
+ # If the user provides an artifact name, there are three possibilities. If
217
+ # an artifact version is also provided with the name, we use both to fetch
218
+ # the artifact version and use it to log the metadata. If no version is
219
+ # provided, if the function is called within a step we search the artifacts
220
+ # of the step if not we fetch the latest version and attach the metadata
221
+ # to the latest version.
222
+ elif artifact_name is not None:
223
+ if artifact_version:
224
+ artifact_version_model = client.get_artifact_version(
225
+ name_id_or_prefix=artifact_name, version=artifact_version
226
+ )
227
+ client.create_run_metadata(
228
+ metadata=metadata,
229
+ resource_id=artifact_version_model.id,
230
+ resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
231
+ )
232
+ else:
233
+ step_context = None
234
+ with contextlib.suppress(RuntimeError):
235
+ step_context = get_step_context()
236
+
237
+ if step_context:
238
+ step_context.add_output_metadata(
239
+ metadata=metadata, output_name=artifact_name
240
+ )
241
+ else:
242
+ artifact_version_model = client.get_artifact_version(
243
+ name_id_or_prefix=artifact_name
244
+ )
245
+ client.create_run_metadata(
246
+ metadata=metadata,
247
+ resource_id=artifact_version_model.id,
248
+ resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
249
+ )
250
+
251
+ # If the user directly provides an artifact_version_id, we use the client to
252
+ # fetch is and attach the metadata accordingly.
253
+ elif artifact_version_id is not None:
254
+ artifact_version_model = client.get_artifact_version(
255
+ name_id_or_prefix=artifact_version_id,
256
+ )
257
+ client.create_run_metadata(
258
+ metadata=metadata,
259
+ resource_id=artifact_version_model.id,
260
+ resource_type=MetadataResourceTypes.ARTIFACT_VERSION,
261
+ )
262
+
263
+ # If every additional value is None, that means we are calling it bare bones
264
+ # and this call needs to happen during a step execution. We will use the
265
+ # step context to fetch the step, run and possibly the model version and
266
+ # attach the metadata accordingly.
267
+ elif all(
268
+ v is None
269
+ for v in [
270
+ step_id,
271
+ step_name,
272
+ run_id_name_or_prefix,
273
+ artifact_version_id,
274
+ artifact_name,
275
+ artifact_version,
276
+ model_version_id,
277
+ model_name,
278
+ model_version,
279
+ ]
280
+ ):
281
+ try:
282
+ step_context = get_step_context()
283
+ except RuntimeError:
284
+ raise ValueError(
285
+ "You are calling 'log_metadata()' outside of a step execution. "
286
+ "If you would like to add metadata to a ZenML entity outside "
287
+ "of the step execution, please provide the required "
288
+ "identifiers."
289
+ )
290
+ client.create_run_metadata(
291
+ metadata=metadata,
292
+ resource_id=step_context.pipeline_run.id,
293
+ resource_type=MetadataResourceTypes.PIPELINE_RUN,
294
+ )
295
+ client.create_run_metadata(
296
+ metadata=metadata,
297
+ resource_id=step_context.step_run.id,
298
+ resource_type=MetadataResourceTypes.STEP_RUN,
299
+ )
300
+ if step_context.model_version:
301
+ client.create_run_metadata(
302
+ metadata=metadata,
303
+ resource_id=step_context.model_version.id,
304
+ resource_type=MetadataResourceTypes.MODEL_VERSION,
305
+ )
306
+
307
+ else:
308
+ raise ValueError(
309
+ """
310
+ Unsupported way to call the `log_metadata`. Possible combinations "
311
+ include:
312
+
313
+ # Inside a step
314
+ # Logs the metadata to the step, its run and possibly its model
315
+ log_metadata(metadata={})
316
+
317
+ # Manually logging for a step
318
+ # Logs the metadata to the step, its run and possibly its model
319
+ log_metadata(metadata={}, step_name=..., run_id_name_or_prefix=...)
320
+ log_metadata(metadata={}, step_id=...)
321
+
322
+ # Manually logging for a run
323
+ # Logs the metadata to the run, possibly its model
324
+ log_metadata(metadata={}, run_id_name_or_prefix=...)
325
+
326
+ # Manually logging for a model
327
+ log_metadata(metadata={}, model_name=..., model_version=...)
328
+ log_metadata(metadata={}, model_version_id=...)
329
+
330
+ # Manually logging for an artifact
331
+ log_metadata(metadata={}, artifact_name=...) # inside a step
332
+ log_metadata(metadata={}, artifact_name=..., artifact_version=...)
333
+ log_metadata(metadata={}, artifact_version_id=...)
334
+ """
335
+ )