zenml-nightly 0.66.0.dev20240928__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (596) hide show
  1. README.md +18 -12
  2. RELEASE_NOTES.md +236 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +5 -10
  5. zenml/analytics/context.py +16 -1
  6. zenml/analytics/enums.py +0 -13
  7. zenml/analytics/utils.py +18 -7
  8. zenml/artifact_stores/base_artifact_store.py +11 -0
  9. zenml/artifacts/artifact_config.py +19 -65
  10. zenml/artifacts/external_artifact.py +7 -53
  11. zenml/artifacts/external_artifact_config.py +21 -40
  12. zenml/artifacts/load_directory_materializer.py +91 -0
  13. zenml/artifacts/utils.py +204 -145
  14. zenml/cli/__init__.py +62 -127
  15. zenml/cli/base.py +8 -8
  16. zenml/cli/integration.py +0 -9
  17. zenml/cli/login.py +951 -0
  18. zenml/cli/pipeline.py +1 -1
  19. zenml/cli/served_model.py +8 -1
  20. zenml/cli/server.py +462 -577
  21. zenml/cli/service_accounts.py +4 -4
  22. zenml/cli/stack.py +78 -569
  23. zenml/cli/stack_components.py +5 -594
  24. zenml/cli/user_management.py +0 -12
  25. zenml/cli/utils.py +24 -142
  26. zenml/client.py +57 -40
  27. zenml/client_lazy_loader.py +29 -12
  28. zenml/config/compiler.py +7 -11
  29. zenml/config/global_config.py +9 -0
  30. zenml/config/pipeline_configurations.py +2 -1
  31. zenml/config/pipeline_run_configuration.py +2 -1
  32. zenml/config/server_config.py +7 -5
  33. zenml/config/step_configurations.py +2 -0
  34. zenml/constants.py +6 -42
  35. zenml/enums.py +1 -4
  36. zenml/environment.py +1 -315
  37. zenml/exceptions.py +12 -41
  38. zenml/hooks/hook_validators.py +1 -8
  39. zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +2 -3
  40. zenml/integrations/argilla/__init__.py +1 -1
  41. zenml/integrations/argilla/annotators/argilla_annotator.py +234 -89
  42. zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +10 -5
  43. zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +9 -5
  44. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +15 -2
  45. zenml/integrations/aws/service_connectors/aws_service_connector.py +6 -0
  46. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +5 -4
  47. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  48. zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +85 -19
  49. zenml/integrations/bentoml/services/__init__.py +15 -4
  50. zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
  51. zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +81 -40
  52. zenml/{steps/base_parameters.py → integrations/bentoml/services/deployment_type.py} +8 -5
  53. zenml/integrations/bentoml/steps/bento_builder.py +1 -0
  54. zenml/integrations/bentoml/steps/bentoml_deployer.py +93 -41
  55. zenml/integrations/github/code_repositories/github_code_repository.py +1 -1
  56. zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +6 -1
  57. zenml/integrations/huggingface/__init__.py +1 -6
  58. zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +3 -3
  59. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +3 -3
  60. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +2 -2
  61. zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +6 -3
  62. zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +6 -0
  63. zenml/integrations/mlflow/services/mlflow_deployment.py +2 -2
  64. zenml/integrations/mlflow/steps/mlflow_deployer.py +1 -1
  65. zenml/integrations/mlflow/steps/mlflow_registry.py +1 -1
  66. zenml/integrations/neptune/experiment_trackers/run_state.py +2 -2
  67. zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
  68. zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
  69. zenml/integrations/s3/artifact_stores/s3_artifact_store.py +98 -0
  70. zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +14 -0
  71. zenml/integrations/s3/utils.py +40 -0
  72. zenml/integrations/tensorflow/__init__.py +0 -9
  73. zenml/logging/step_logging.py +13 -15
  74. zenml/{new/pipelines → login}/__init__.py +4 -1
  75. zenml/login/credentials.py +346 -0
  76. zenml/login/credentials_store.py +603 -0
  77. zenml/{new → login/pro}/__init__.py +4 -1
  78. zenml/login/pro/client.py +496 -0
  79. zenml/login/pro/constants.py +34 -0
  80. zenml/{lineage_graph/edge.py → login/pro/models.py} +9 -8
  81. zenml/{zen_server/deploy/terraform/providers → login/pro/organization}/__init__.py +2 -2
  82. zenml/login/pro/organization/client.py +79 -0
  83. zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
  84. zenml/{new/steps → login/pro/tenant}/__init__.py +2 -1
  85. zenml/login/pro/tenant/client.py +92 -0
  86. zenml/login/pro/tenant/models.py +174 -0
  87. zenml/login/pro/utils.py +121 -0
  88. zenml/{cli → login}/web_login.py +64 -28
  89. zenml/materializers/__init__.py +2 -0
  90. zenml/materializers/base_materializer.py +43 -9
  91. zenml/materializers/built_in_materializer.py +1 -1
  92. zenml/materializers/uuid_materializer.py +79 -0
  93. zenml/metadata/lazy_load.py +10 -7
  94. zenml/metadata/metadata_types.py +49 -0
  95. zenml/model/lazy_load.py +85 -4
  96. zenml/model/model.py +61 -83
  97. zenml/model/utils.py +47 -118
  98. zenml/models/__init__.py +6 -0
  99. zenml/models/v2/base/base.py +12 -8
  100. zenml/models/v2/base/filter.py +9 -0
  101. zenml/models/v2/core/artifact_version.py +53 -25
  102. zenml/models/v2/core/component.py +54 -19
  103. zenml/models/v2/core/flavor.py +13 -13
  104. zenml/models/v2/core/model.py +4 -1
  105. zenml/models/v2/core/model_version.py +4 -6
  106. zenml/models/v2/core/model_version_artifact.py +3 -1
  107. zenml/models/v2/core/model_version_pipeline_run.py +3 -1
  108. zenml/models/v2/core/pipeline.py +3 -1
  109. zenml/models/v2/core/pipeline_run.py +24 -3
  110. zenml/models/v2/core/run_metadata.py +2 -2
  111. zenml/models/v2/core/run_template.py +3 -1
  112. zenml/models/v2/core/stack.py +7 -3
  113. zenml/models/v2/core/step_run.py +58 -14
  114. zenml/models/v2/misc/auth_models.py +11 -2
  115. zenml/models/v2/misc/server_models.py +2 -4
  116. zenml/orchestrators/base_orchestrator.py +41 -4
  117. zenml/orchestrators/input_utils.py +52 -36
  118. zenml/orchestrators/step_launcher.py +50 -171
  119. zenml/orchestrators/step_run_utils.py +574 -0
  120. zenml/orchestrators/step_runner.py +83 -120
  121. zenml/orchestrators/utils.py +46 -180
  122. zenml/pipelines/__init__.py +3 -16
  123. zenml/{new/pipelines → pipelines}/pipeline_context.py +3 -18
  124. zenml/pipelines/pipeline_decorator.py +34 -60
  125. zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +210 -207
  126. zenml/{new/pipelines → pipelines}/run_utils.py +11 -10
  127. zenml/services/local/local_daemon_entrypoint.py +4 -4
  128. zenml/services/service.py +2 -2
  129. zenml/stack/stack.py +3 -108
  130. zenml/stack/stack_component.py +2 -78
  131. zenml/stack/utils.py +26 -14
  132. zenml/stack_deployments/aws_stack_deployment.py +3 -0
  133. zenml/step_operators/step_operator_entrypoint_configuration.py +1 -1
  134. zenml/steps/__init__.py +3 -9
  135. zenml/steps/base_step.py +25 -267
  136. zenml/steps/entrypoint_function_utils.py +2 -80
  137. zenml/{new/steps → steps}/step_context.py +12 -66
  138. zenml/steps/step_decorator.py +42 -89
  139. zenml/steps/step_invocation.py +16 -60
  140. zenml/steps/utils.py +4 -23
  141. zenml/utils/dashboard_utils.py +16 -51
  142. zenml/utils/materializer_utils.py +1 -1
  143. zenml/utils/requirements_utils.py +71 -0
  144. zenml/utils/singleton.py +15 -3
  145. zenml/utils/source_utils.py +39 -2
  146. zenml/utils/typing_utils.py +8 -28
  147. zenml/utils/visualization_utils.py +1 -1
  148. zenml/zen_server/auth.py +44 -39
  149. zenml/zen_server/dashboard/assets/{404-Y50hSt65.js → 404-DT4QRUqN.js} +1 -1
  150. zenml/zen_server/dashboard/assets/@radix-DP6vWzyx.js +85 -0
  151. zenml/zen_server/dashboard/assets/@react-router-BMhZulnd.js +29 -0
  152. zenml/zen_server/dashboard/assets/@reactflow-8U9qNlMR.js +17 -0
  153. zenml/zen_server/dashboard/assets/@tanstack-BUCbhJyH.js +22 -0
  154. zenml/zen_server/dashboard/assets/{AlertDialogDropdownItem-xLR9a1iw.js → AlertDialogDropdownItem-C6N2rGrB.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{CodeSnippet-IxXNxUDa.js → CodeSnippet-CqybNv0k.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{CollapsibleCard-BhutZbBL.js → CollapsibleCard-0r_8G2Lj.js} +1 -1
  157. zenml/zen_server/dashboard/assets/{Commands-Bf-rd1z8.js → Commands-BDjgBQKi.js} +1 -1
  158. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +1 -0
  159. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +1 -0
  160. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +1 -0
  161. zenml/zen_server/dashboard/assets/{CopyButton-DcFHidFJ.js → CopyButton-C745BrKi.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{CsvVizualization-QSbjrfxw.js → CsvVizualization-PpAq0CeZ.js} +5 -5
  163. zenml/zen_server/dashboard/assets/{DialogItem-Cd3HqST4.js → DialogItem-DcVCZEew.js} +1 -1
  164. zenml/zen_server/dashboard/assets/{DisplayDate-DkCy54Bp.js → DisplayDate-BeXgUG_C.js} +1 -1
  165. zenml/zen_server/dashboard/assets/{EmptyState-Cs3DEmso.js → EmptyState-DeK7H4pr.js} +1 -1
  166. zenml/zen_server/dashboard/assets/{Error-BhwdmqK7.js → Error-BMlzibXj.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{Helpbox-0pBpTwTm.js → Helpbox-BLf40fLV.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{Infobox-BTK_EUKT.js → Infobox-BwisKifi.js} +1 -1
  169. zenml/zen_server/dashboard/assets/{InlineAvatar-CA3DFMcM.js → InlineAvatar-jEgodSgX.js} +1 -1
  170. zenml/zen_server/dashboard/assets/{Lock-CmIn0szs.js → Lock-3lLt1ih0.js} +1 -1
  171. zenml/zen_server/dashboard/assets/MarkdownVisualization-8O9kTr-2.js +14 -0
  172. zenml/zen_server/dashboard/assets/{NumberBox-CrN0_kqI.js → NumberBox-T9eELfLZ.js} +1 -1
  173. zenml/zen_server/dashboard/assets/{Partials-QLOZw624.js → Partials-YPBB3V4q.js} +1 -1
  174. zenml/zen_server/dashboard/assets/{PasswordChecker-DE71J_3F.js → PasswordChecker-CW0kqY0W.js} +1 -1
  175. zenml/zen_server/dashboard/assets/{ProviderIcon-C16CCIN4.js → ProviderIcon-Bb3Xha5A.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{ProviderRadio-D3FuCHf3.js → ProviderRadio-BROY1700.js} +1 -1
  177. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +1 -0
  178. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +1 -0
  179. zenml/zen_server/dashboard/assets/{SecretTooltip-BaMwHF-Q.js → SecretTooltip-C_qByGWB.js} +1 -1
  180. zenml/zen_server/dashboard/assets/{SetPassword-DuIC65H9.js → SetPassword-7pRB00El.js} +1 -1
  181. zenml/zen_server/dashboard/assets/{Tick-DJTCF0Re.js → Tick-CykQFPj2.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CUm-DMpw.js → UpdatePasswordSchemas-DckMEkFf.js} +1 -1
  183. zenml/zen_server/dashboard/assets/{UsageReason-CKw0juLF.js → UsageReason-DVceN14P.js} +1 -1
  184. zenml/zen_server/dashboard/assets/{WizardFooter-Cv9ApYWU.js → WizardFooter-CW0Cvd70.js} +1 -1
  185. zenml/zen_server/dashboard/assets/{check-circle-DOoS4yhF.js → check-circle-Dwxliy1Z.js} +1 -1
  186. zenml/zen_server/dashboard/assets/{chevron-down-Cwb-W_B_.js → chevron-down-8wLBS5pQ.js} +1 -1
  187. zenml/zen_server/dashboard/assets/{chevron-right-double-c9H46Kl8.js → chevron-right-double-DoD8iXWM.js} +1 -1
  188. zenml/zen_server/dashboard/assets/{cloud-only-BB4BVa6E.js → cloud-only-B-s_HMDm.js} +1 -1
  189. zenml/zen_server/dashboard/assets/{code-browser-BJYErIjr.js → code-browser-CZUQs3Wa.js} +1 -1
  190. zenml/zen_server/dashboard/assets/{copy-CaGlDsUy.js → copy-CaSMXwiU.js} +1 -1
  191. zenml/zen_server/dashboard/assets/{create-stack-F29xAUEx.js → create-stack-CEmaPZ4c.js} +1 -1
  192. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +1 -0
  193. zenml/zen_server/dashboard/assets/{docker-BFAFXr2_.js → docker-BFNgg-z3.js} +1 -1
  194. zenml/zen_server/dashboard/assets/{dots-horizontal-C6K59vUm.js → dots-horizontal-DK5Duzx4.js} +1 -1
  195. zenml/zen_server/dashboard/assets/{form-schemas-BKXwSDK2.js → form-schemas-1AyOCx90.js} +1 -1
  196. zenml/zen_server/dashboard/assets/{gcp-Dj6ntk0L.js → gcp-7M2Yf3ZK.js} +1 -1
  197. zenml/zen_server/dashboard/assets/{help-CwN931fX.js → help-Dam461dC.js} +1 -1
  198. zenml/zen_server/dashboard/assets/{index-5GJ5ysEZ.js → index-BVJ8n2-j.js} +1 -1
  199. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +1 -0
  200. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +1 -0
  201. zenml/zen_server/dashboard/assets/{index-D-mtoBj3.js → index-DKPhqP2B.js} +1 -1
  202. zenml/zen_server/dashboard/assets/index-QQb7wQEC.js +55 -0
  203. zenml/zen_server/dashboard/assets/index.esm-cuVep_NJ.js +1 -0
  204. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
  205. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
  206. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
  207. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
  208. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
  209. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
  210. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
  211. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
  212. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
  213. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
  214. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
  215. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
  216. zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
  217. zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
  218. zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
  219. zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
  220. zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
  221. zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
  222. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
  223. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
  224. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
  225. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
  226. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
  227. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
  228. zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
  229. zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
  230. zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
  231. zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
  232. zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
  233. zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
  234. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
  235. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
  236. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
  237. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
  238. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
  239. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
  240. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
  241. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
  242. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
  243. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
  244. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
  245. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
  246. zenml/zen_server/dashboard/assets/{kubernetes-BjbR6D-1.js → kubernetes--g7r02Zu.js} +1 -1
  247. zenml/zen_server/dashboard/assets/{layout-Dru15_XR.js → layout-DCSYN7-C.js} +1 -1
  248. zenml/zen_server/dashboard/assets/{link-external-BT2L8hAQ.js → link-external-CBEk6kEG.js} +1 -1
  249. zenml/zen_server/dashboard/assets/{login-mutation-ax6iL2Mb.js → login-mutation-DTcAFP1l.js} +1 -1
  250. zenml/zen_server/dashboard/assets/{logs-GiDJXbLS.js → logs-D5bdJGur.js} +1 -1
  251. zenml/zen_server/dashboard/assets/not-found-Cc-JkRH2.js +1 -0
  252. zenml/zen_server/dashboard/assets/{package-DYKZ5jKW.js → package-Cs35Szwh.js} +1 -1
  253. zenml/zen_server/dashboard/assets/{page-HkVBdZl6.js → page-9yplj5JT.js} +1 -1
  254. zenml/zen_server/dashboard/assets/{page-uORspyRu.js → page-B-y2XKIc.js} +1 -1
  255. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +1 -0
  256. zenml/zen_server/dashboard/assets/{page-BDg1F-Ug.js → page-BQQKaabe.js} +2 -2
  257. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +1 -0
  258. zenml/zen_server/dashboard/assets/{page-CvKnNK1S.js → page-BjjlMk7s.js} +1 -1
  259. zenml/zen_server/dashboard/assets/{page-BukXK1Aa.js → page-Bvd7YH2A.js} +1 -1
  260. zenml/zen_server/dashboard/assets/{page-DcTjHmYZ.js → page-C0N5q3l7.js} +1 -1
  261. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +1 -0
  262. zenml/zen_server/dashboard/assets/{page-DuqYMYmH.js → page-C_f47pBf.js} +1 -1
  263. zenml/zen_server/dashboard/assets/{page-DRfcRK1w.js → page-ClUVkl-O.js} +1 -1
  264. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +3 -0
  265. zenml/zen_server/dashboard/assets/{page-p0BhSAWx.js → page-CyN2bdWG.js} +1 -1
  266. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +1 -0
  267. zenml/zen_server/dashboard/assets/{page-6mfzecin.js → page-DH_Z7iW1.js} +1 -1
  268. zenml/zen_server/dashboard/assets/{page-8kYmrh0B.js → page-DIOXwhiD.js} +1 -1
  269. zenml/zen_server/dashboard/assets/{page-BXarY9K2.js → page-DLixvR-7.js} +1 -1
  270. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +1 -0
  271. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +1 -0
  272. zenml/zen_server/dashboard/assets/{page-MAXyfXBq.js → page-Ddgy6kDS.js} +1 -1
  273. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +9 -0
  274. zenml/zen_server/dashboard/assets/{page-CKHNAq7z.js → page-Dx16z7nA.js} +1 -1
  275. zenml/zen_server/dashboard/assets/{page-DMSLXKGT.js → page-DzpVUZ8f.js} +1 -1
  276. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +1 -0
  277. zenml/zen_server/dashboard/assets/{page-miU2rhYG.js → page-N4qoPHKb.js} +1 -1
  278. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +1 -0
  279. zenml/zen_server/dashboard/assets/{page-3qPX9WYH.js → page-bKaULTGG.js} +1 -1
  280. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +1 -0
  281. zenml/zen_server/dashboard/assets/{persist-CfJMar_k.js → persist-CP0JmYZ4.js} +1 -1
  282. zenml/zen_server/dashboard/assets/{persist-BxIR2XZs.js → persist-DNb5cdrU.js} +1 -1
  283. zenml/zen_server/dashboard/assets/{plus-Bc8eLSDM.js → plus-C9IxgN2M.js} +1 -1
  284. zenml/zen_server/dashboard/assets/{refresh-hfgWPeto.js → refresh-BVu22P_C.js} +1 -1
  285. zenml/zen_server/dashboard/assets/{rocket-SESCGQXm.js → rocket-CONEmRmB.js} +1 -1
  286. zenml/zen_server/dashboard/assets/{sharedSchema-vub0rii3.js → sharedSchema-TMLu-nYQ.js} +1 -1
  287. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +1 -0
  288. zenml/zen_server/dashboard/assets/{tick-circle-m-hJG8i9.js → tick-circle-CM1ZScbQ.js} +1 -1
  289. zenml/zen_server/dashboard/assets/{trash-DUWZWzse.js → trash-DkJHMOg7.js} +1 -1
  290. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-FGVP7X2U.js → update-server-settings-mutation-DsU8cNVl.js} +1 -1
  291. zenml/zen_server/dashboard/assets/{url-CbAPzsmT.js → url-D5le3J4q.js} +1 -1
  292. zenml/zen_server/dashboard/assets/zod-D89GC_vc.js +1 -0
  293. zenml/zen_server/dashboard/index.html +7 -7
  294. zenml/zen_server/deploy/__init__.py +7 -16
  295. zenml/zen_server/deploy/base_provider.py +46 -73
  296. zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
  297. zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +44 -63
  298. zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +50 -28
  299. zenml/zen_server/deploy/deployer.py +90 -171
  300. zenml/zen_server/deploy/deployment.py +20 -12
  301. zenml/zen_server/deploy/docker/docker_provider.py +9 -28
  302. zenml/zen_server/deploy/docker/docker_zen_server.py +19 -9
  303. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  304. zenml/zen_server/deploy/helm/README.md +2 -2
  305. zenml/zen_server/exceptions.py +11 -0
  306. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +1 -1
  307. zenml/zen_server/jwt.py +9 -9
  308. zenml/zen_server/routers/artifact_version_endpoints.py +1 -1
  309. zenml/zen_server/routers/auth_endpoints.py +30 -8
  310. zenml/zen_server/routers/runs_endpoints.py +0 -28
  311. zenml/zen_server/routers/stack_components_endpoints.py +1 -1
  312. zenml/zen_server/routers/workspaces_endpoints.py +1 -1
  313. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +7 -4
  314. zenml/zen_server/template_execution/utils.py +8 -63
  315. zenml/zen_server/utils.py +67 -69
  316. zenml/zen_server/zen_server_api.py +107 -17
  317. zenml/zen_stores/base_zen_store.py +4 -51
  318. zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
  319. zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
  320. zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
  321. zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
  322. zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
  323. zenml/zen_stores/rest_zen_store.py +325 -147
  324. zenml/zen_stores/schemas/api_key_schemas.py +9 -4
  325. zenml/zen_stores/schemas/artifact_schemas.py +21 -2
  326. zenml/zen_stores/schemas/artifact_visualization_schemas.py +1 -1
  327. zenml/zen_stores/schemas/component_schemas.py +49 -6
  328. zenml/zen_stores/schemas/device_schemas.py +9 -4
  329. zenml/zen_stores/schemas/flavor_schemas.py +1 -1
  330. zenml/zen_stores/schemas/model_schemas.py +1 -1
  331. zenml/zen_stores/schemas/pipeline_run_schemas.py +12 -4
  332. zenml/zen_stores/schemas/service_schemas.py +1 -1
  333. zenml/zen_stores/schemas/step_run_schemas.py +3 -3
  334. zenml/zen_stores/schemas/trigger_schemas.py +1 -1
  335. zenml/zen_stores/sql_zen_store.py +422 -145
  336. zenml/zen_stores/template_utils.py +3 -1
  337. {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/METADATA +20 -18
  338. {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/RECORD +343 -505
  339. {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/WHEEL +1 -1
  340. zenml/api.py +0 -60
  341. zenml/cli/stack_recipes.py +0 -469
  342. zenml/lineage_graph/__init__.py +0 -34
  343. zenml/lineage_graph/lineage_graph.py +0 -244
  344. zenml/lineage_graph/node/__init__.py +0 -32
  345. zenml/lineage_graph/node/artifact_node.py +0 -51
  346. zenml/lineage_graph/node/step_node.py +0 -41
  347. zenml/model/model_version.py +0 -38
  348. zenml/new/pipelines/code_archive.py +0 -157
  349. zenml/new/pipelines/pipeline_decorator.py +0 -118
  350. zenml/new/steps/step_decorator.py +0 -176
  351. zenml/pipelines/base_pipeline.py +0 -273
  352. zenml/post_execution/__init__.py +0 -30
  353. zenml/post_execution/pipeline.py +0 -55
  354. zenml/post_execution/pipeline_run.py +0 -55
  355. zenml/services/terraform/__init__.py +0 -14
  356. zenml/services/terraform/terraform_service.py +0 -441
  357. zenml/steps/external_artifact.py +0 -26
  358. zenml/steps/step_environment.py +0 -108
  359. zenml/steps/step_output.py +0 -36
  360. zenml/utils/mlstacks_utils.py +0 -635
  361. zenml/utils/terraform_utils.py +0 -43
  362. zenml/zen_server/dashboard/assets/@radix-DnFH_oo1.js +0 -85
  363. zenml/zen_server/dashboard/assets/@react-router-APVeuk-U.js +0 -29
  364. zenml/zen_server/dashboard/assets/@reactflow-ytavUpwh.js +0 -17
  365. zenml/zen_server/dashboard/assets/@tanstack-QbMbTrh5.js +0 -22
  366. zenml/zen_server/dashboard/assets/ComponentBadge-gKR1OIwG.js +0 -1
  367. zenml/zen_server/dashboard/assets/ExecutionStatus-D6r6aK8J.js +0 -1
  368. zenml/zen_server/dashboard/assets/MarkdownVisualization-DS05sfBm.js +0 -14
  369. zenml/zen_server/dashboard/assets/SearchField-BzmfxS0L.js +0 -1
  370. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BA3R2Sey.js +0 -1
  371. zenml/zen_server/dashboard/assets/delete-run-CP0pcJ3U.js +0 -1
  372. zenml/zen_server/dashboard/assets/index-BhJ6ZJxv.css +0 -1
  373. zenml/zen_server/dashboard/assets/index-Ci0nJ8EZ.js +0 -55
  374. zenml/zen_server/dashboard/assets/index.esm-BE1uqCX5.js +0 -1
  375. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-Df6ckaLK.woff2 +0 -0
  376. zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-JrS_4yms.woff +0 -0
  377. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-BRrLR67x.woff2 +0 -0
  378. zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DskEQOpE.woff +0 -0
  379. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-DDpWG8g5.woff2 +0 -0
  380. zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-iz1--dBq.woff +0 -0
  381. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-CzG7Kr3z.woff +0 -0
  382. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-tyfMZHQw.woff2 +0 -0
  383. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-DOnSzjnx.woff2 +0 -0
  384. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-Xebo2OyJ.woff +0 -0
  385. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-DpA2xaRd.woff2 +0 -0
  386. zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-KAwcVx6H.woff +0 -0
  387. zenml/zen_server/dashboard/assets/inter-greek-400-normal-DQXyrmoy.woff2 +0 -0
  388. zenml/zen_server/dashboard/assets/inter-greek-400-normal-DvIPHDQ7.woff +0 -0
  389. zenml/zen_server/dashboard/assets/inter-greek-500-normal-BjpBGs91.woff +0 -0
  390. zenml/zen_server/dashboard/assets/inter-greek-500-normal-CmOavsDc.woff2 +0 -0
  391. zenml/zen_server/dashboard/assets/inter-greek-600-normal-B-l8Lzzd.woff2 +0 -0
  392. zenml/zen_server/dashboard/assets/inter-greek-600-normal-BLZsI-P3.woff +0 -0
  393. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-CIdlr5YK.woff2 +0 -0
  394. zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-_Rr29XE2.woff +0 -0
  395. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-2pdUafRD.woff2 +0 -0
  396. zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-Dtavx3qw.woff +0 -0
  397. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-BmtRFZgT.woff +0 -0
  398. zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-CkBLDEl_.woff2 +0 -0
  399. zenml/zen_server/dashboard/assets/inter-latin-400-normal-BT1H-PT_.woff2 +0 -0
  400. zenml/zen_server/dashboard/assets/inter-latin-400-normal-Cdi8t5Mu.woff +0 -0
  401. zenml/zen_server/dashboard/assets/inter-latin-500-normal-D4I8BKCx.woff +0 -0
  402. zenml/zen_server/dashboard/assets/inter-latin-500-normal-kWhwEdDH.woff2 +0 -0
  403. zenml/zen_server/dashboard/assets/inter-latin-600-normal-B2Ssfs8e.woff2 +0 -0
  404. zenml/zen_server/dashboard/assets/inter-latin-600-normal-Dbvh0wvx.woff +0 -0
  405. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-8tIzm-yw.woff +0 -0
  406. zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-D3W-OpO-.woff2 +0 -0
  407. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-B9u8Q_zH.woff2 +0 -0
  408. zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-SuUkSNTU.woff +0 -0
  409. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-Dg0Bk0Yr.woff +0 -0
  410. zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-ao35dkSb.woff2 +0 -0
  411. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-Cnt0N5Vm.woff2 +0 -0
  412. zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DIOGfGLL.woff +0 -0
  413. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-0i6yoQMg.woff +0 -0
  414. zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-5IkPJ6Nk.woff2 +0 -0
  415. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-BQ_dbMbg.woff +0 -0
  416. zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-DjnxGF-L.woff2 +0 -0
  417. zenml/zen_server/dashboard/assets/not-found-DbjllLY_.js +0 -1
  418. zenml/zen_server/dashboard/assets/page-B1n7_W7z.js +0 -1
  419. zenml/zen_server/dashboard/assets/page-BZZhLo2u.js +0 -1
  420. zenml/zen_server/dashboard/assets/page-Bbf_oBjn.js +0 -1
  421. zenml/zen_server/dashboard/assets/page-BjjuBvZG.js +0 -9
  422. zenml/zen_server/dashboard/assets/page-CHaQkFK5.js +0 -1
  423. zenml/zen_server/dashboard/assets/page-CS0SYFK8.js +0 -1
  424. zenml/zen_server/dashboard/assets/page-DGM1CbYT.js +0 -2
  425. zenml/zen_server/dashboard/assets/page-DOmIZ2ra.js +0 -1
  426. zenml/zen_server/dashboard/assets/page-DYVmJ9_w.js +0 -3
  427. zenml/zen_server/dashboard/assets/page-Dwow2doB.js +0 -1
  428. zenml/zen_server/dashboard/assets/stack-detail-query-DQcyzG-2.js +0 -1
  429. zenml/zen_server/dashboard/assets/zod-uFd1wBcd.js +0 -1
  430. zenml/zen_server/dashboard_legacy/_redirects +0 -1
  431. zenml/zen_server/dashboard_legacy/asset-manifest.json +0 -131
  432. zenml/zen_server/dashboard_legacy/favicon.ico +0 -0
  433. zenml/zen_server/dashboard_legacy/index.html +0 -1
  434. zenml/zen_server/dashboard_legacy/manifest.json +0 -25
  435. zenml/zen_server/dashboard_legacy/precache-manifest.6d320abb70db612019dda6c4948e7a90.js +0 -462
  436. zenml/zen_server/dashboard_legacy/robots.txt +0 -2
  437. zenml/zen_server/dashboard_legacy/service-worker.js +0 -39
  438. zenml/zen_server/dashboard_legacy/static/css/2.5b37d44a.chunk.css +0 -16
  439. zenml/zen_server/dashboard_legacy/static/css/2.5b37d44a.chunk.css.map +0 -1
  440. zenml/zen_server/dashboard_legacy/static/css/main.e79dff04.chunk.css +0 -2
  441. zenml/zen_server/dashboard_legacy/static/css/main.e79dff04.chunk.css.map +0 -1
  442. zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js +0 -3
  443. zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js.LICENSE.txt +0 -95
  444. zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js.map +0 -1
  445. zenml/zen_server/dashboard_legacy/static/js/main.fa9299d5.chunk.js +0 -2
  446. zenml/zen_server/dashboard_legacy/static/js/main.fa9299d5.chunk.js.map +0 -1
  447. zenml/zen_server/dashboard_legacy/static/js/runtime-main.bfca2edd.js +0 -2
  448. zenml/zen_server/dashboard_legacy/static/js/runtime-main.bfca2edd.js.map +0 -1
  449. zenml/zen_server/dashboard_legacy/static/media/AlertTriangle.28aee535.svg +0 -5
  450. zenml/zen_server/dashboard_legacy/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
  451. zenml/zen_server/dashboard_legacy/static/media/Back.86c23a22.svg +0 -4
  452. zenml/zen_server/dashboard_legacy/static/media/BookOpen.5cb101ff.svg +0 -4
  453. zenml/zen_server/dashboard_legacy/static/media/BoundingBox.1eb98717.svg +0 -10
  454. zenml/zen_server/dashboard_legacy/static/media/Burger.9b1c67d7.svg +0 -3
  455. zenml/zen_server/dashboard_legacy/static/media/Cached.2381fb8d.svg +0 -1
  456. zenml/zen_server/dashboard_legacy/static/media/Calendar.356e11c7.svg +0 -3
  457. zenml/zen_server/dashboard_legacy/static/media/ChartBarHorizontal.0247447b.svg +0 -6
  458. zenml/zen_server/dashboard_legacy/static/media/ChartLine.0d79e18d.svg +0 -4
  459. zenml/zen_server/dashboard_legacy/static/media/ChatDots.2e1c9211.svg +0 -6
  460. zenml/zen_server/dashboard_legacy/static/media/Check.dad6beb2.svg +0 -3
  461. zenml/zen_server/dashboard_legacy/static/media/CheckCircleFilled.c19566d0.svg +0 -3
  462. zenml/zen_server/dashboard_legacy/static/media/Checkbox.af50e31e.svg +0 -3
  463. zenml/zen_server/dashboard_legacy/static/media/ChevronDown.f860ce32.svg +0 -3
  464. zenml/zen_server/dashboard_legacy/static/media/ChevronDownLight.6642d756.svg +0 -3
  465. zenml/zen_server/dashboard_legacy/static/media/ChevronLeft.f6edfcdb.svg +0 -3
  466. zenml/zen_server/dashboard_legacy/static/media/CircleCheck.f98fd6ca.svg +0 -1
  467. zenml/zen_server/dashboard_legacy/static/media/Clock.ffc9de95.svg +0 -3
  468. zenml/zen_server/dashboard_legacy/static/media/Close.74e9efbc.svg +0 -5
  469. zenml/zen_server/dashboard_legacy/static/media/CloseWithBorder.6960930a.svg +0 -3
  470. zenml/zen_server/dashboard_legacy/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
  471. zenml/zen_server/dashboard_legacy/static/media/CloudArrowUp.0aecb235.svg +0 -6
  472. zenml/zen_server/dashboard_legacy/static/media/Code.ef0f33b5.svg +0 -3
  473. zenml/zen_server/dashboard_legacy/static/media/Config.0be63f8a.svg +0 -1
  474. zenml/zen_server/dashboard_legacy/static/media/Connector.9fd46ef1.svg +0 -10
  475. zenml/zen_server/dashboard_legacy/static/media/Copy.36e2112a.svg +0 -1
  476. zenml/zen_server/dashboard_legacy/static/media/Dashboard.d05787e0.svg +0 -3
  477. zenml/zen_server/dashboard_legacy/static/media/Data.b1c3b5f8.svg +0 -3
  478. zenml/zen_server/dashboard_legacy/static/media/Delete.3c361b28.svg +0 -8
  479. zenml/zen_server/dashboard_legacy/static/media/Docs.7541d478.svg +0 -7
  480. zenml/zen_server/dashboard_legacy/static/media/Download.fba04d87.svg +0 -5
  481. zenml/zen_server/dashboard_legacy/static/media/Edit.490eb294.svg +0 -6
  482. zenml/zen_server/dashboard_legacy/static/media/EmptyRightArrow.23749d01.svg +0 -3
  483. zenml/zen_server/dashboard_legacy/static/media/Example.6396cd37.svg +0 -5
  484. zenml/zen_server/dashboard_legacy/static/media/Extension.1394cd4a.svg +0 -3
  485. zenml/zen_server/dashboard_legacy/static/media/Eye.d9e4ee62.svg +0 -4
  486. zenml/zen_server/dashboard_legacy/static/media/Failed.0213c1a0.svg +0 -1
  487. zenml/zen_server/dashboard_legacy/static/media/FileText.1f15bacd.svg +0 -7
  488. zenml/zen_server/dashboard_legacy/static/media/Filter.ab6b9c0d.svg +0 -3
  489. zenml/zen_server/dashboard_legacy/static/media/Folders.12b29887.svg +0 -5
  490. zenml/zen_server/dashboard_legacy/static/media/FunnelFill.6df4c143.svg +0 -3
  491. zenml/zen_server/dashboard_legacy/static/media/GitCommit.7dd9c2aa.svg +0 -5
  492. zenml/zen_server/dashboard_legacy/static/media/GitHub_Logo.cefc2023.png +0 -0
  493. zenml/zen_server/dashboard_legacy/static/media/Graph.2c63a892.svg +0 -11
  494. zenml/zen_server/dashboard_legacy/static/media/History.08329240.svg +0 -3
  495. zenml/zen_server/dashboard_legacy/static/media/Home.0843b0d5.svg +0 -3
  496. zenml/zen_server/dashboard_legacy/static/media/ImageBuilder.ea762d9c.svg +0 -6
  497. zenml/zen_server/dashboard_legacy/static/media/InProgress.304a0edc.svg +0 -1
  498. zenml/zen_server/dashboard_legacy/static/media/Info.9fe10c5c.svg +0 -3
  499. zenml/zen_server/dashboard_legacy/static/media/KeyboardReturn.491afbe3.svg +0 -3
  500. zenml/zen_server/dashboard_legacy/static/media/Link.72bbb55d.svg +0 -4
  501. zenml/zen_server/dashboard_legacy/static/media/Lock.30f5e1fe.svg +0 -5
  502. zenml/zen_server/dashboard_legacy/static/media/Lock2.a769ea52.svg +0 -3
  503. zenml/zen_server/dashboard_legacy/static/media/LockKey.92f21621.svg +0 -6
  504. zenml/zen_server/dashboard_legacy/static/media/Logs.8bf4d005.svg +0 -5
  505. zenml/zen_server/dashboard_legacy/static/media/MinusCircle.4188f418.svg +0 -4
  506. zenml/zen_server/dashboard_legacy/static/media/ModelRegistry.f0de050a.svg +0 -6
  507. zenml/zen_server/dashboard_legacy/static/media/MultiUser.a2ba7c67.svg +0 -10
  508. zenml/zen_server/dashboard_legacy/static/media/PaginationFirst.92628634.svg +0 -4
  509. zenml/zen_server/dashboard_legacy/static/media/PaginationLast.00d3c732.svg +0 -4
  510. zenml/zen_server/dashboard_legacy/static/media/PaginationNext.86158845.svg +0 -3
  511. zenml/zen_server/dashboard_legacy/static/media/PaginationPrev.60c18a88.svg +0 -3
  512. zenml/zen_server/dashboard_legacy/static/media/Pen.f2d831d4.svg +0 -6
  513. zenml/zen_server/dashboard_legacy/static/media/PhotoCamera.179d6d4c.svg +0 -3
  514. zenml/zen_server/dashboard_legacy/static/media/Pipeline.30d298b0.svg +0 -7
  515. zenml/zen_server/dashboard_legacy/static/media/Plus.5aa1c16b.svg +0 -3
  516. zenml/zen_server/dashboard_legacy/static/media/PlusCircle.92d860dd.svg +0 -5
  517. zenml/zen_server/dashboard_legacy/static/media/Repositories.71a36b8c.svg +0 -3
  518. zenml/zen_server/dashboard_legacy/static/media/RightArrow.f30d3871.svg +0 -29
  519. zenml/zen_server/dashboard_legacy/static/media/Rocket.63bf7b9d.svg +0 -3
  520. zenml/zen_server/dashboard_legacy/static/media/RocketLaunch.1bff2b59.svg +0 -6
  521. zenml/zen_server/dashboard_legacy/static/media/Rubik-Medium.c87313aa.ttf +0 -0
  522. zenml/zen_server/dashboard_legacy/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
  523. zenml/zen_server/dashboard_legacy/static/media/Run.daec4fb2.svg +0 -6
  524. zenml/zen_server/dashboard_legacy/static/media/Search.d1afcce5.svg +0 -4
  525. zenml/zen_server/dashboard_legacy/static/media/Settings.59ca73ae.svg +0 -4
  526. zenml/zen_server/dashboard_legacy/static/media/Share2.46c3ff66.svg +0 -3
  527. zenml/zen_server/dashboard_legacy/static/media/SignOut.6aa718c5.svg +0 -3
  528. zenml/zen_server/dashboard_legacy/static/media/SimplePlus.5cf7ec20.svg +0 -3
  529. zenml/zen_server/dashboard_legacy/static/media/SingleUser.bef3a095.svg +0 -4
  530. zenml/zen_server/dashboard_legacy/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
  531. zenml/zen_server/dashboard_legacy/static/media/Stack.19b604ac.svg +0 -5
  532. zenml/zen_server/dashboard_legacy/static/media/StackComponent.b1ba90b5.svg +0 -4
  533. zenml/zen_server/dashboard_legacy/static/media/Star.f0c25022.svg +0 -9
  534. zenml/zen_server/dashboard_legacy/static/media/StarOutline.94ca8cd9.svg +0 -3
  535. zenml/zen_server/dashboard_legacy/static/media/Storefront.4b4796fe.svg +0 -3
  536. zenml/zen_server/dashboard_legacy/static/media/Stream.543e3039.svg +0 -3
  537. zenml/zen_server/dashboard_legacy/static/media/SupportAgent.510ddf1f.svg +0 -8
  538. zenml/zen_server/dashboard_legacy/static/media/Table.77033750.svg +0 -6
  539. zenml/zen_server/dashboard_legacy/static/media/Tool.d5785486.svg +0 -3
  540. zenml/zen_server/dashboard_legacy/static/media/UserPlus.741a99d7.svg +0 -6
  541. zenml/zen_server/dashboard_legacy/static/media/Verified.0625b2a0.svg +0 -3
  542. zenml/zen_server/dashboard_legacy/static/media/addNew.4fb6c939.svg +0 -8
  543. zenml/zen_server/dashboard_legacy/static/media/arrowClose.cbd53f3f.svg +0 -3
  544. zenml/zen_server/dashboard_legacy/static/media/arrowOpen.6ceef0af.svg +0 -3
  545. zenml/zen_server/dashboard_legacy/static/media/check_small.30bc0138.svg +0 -3
  546. zenml/zen_server/dashboard_legacy/static/media/circleArrowSideClose.98d6013e.svg +0 -18
  547. zenml/zen_server/dashboard_legacy/static/media/circleArrowSideOpen.63653df6.svg +0 -18
  548. zenml/zen_server/dashboard_legacy/static/media/image.104fd14b.png +0 -0
  549. zenml/zen_server/dashboard_legacy/static/media/imageAddIcon.e83004a9.svg +0 -7
  550. zenml/zen_server/dashboard_legacy/static/media/logo.93333e5c.svg +0 -1
  551. zenml/zen_server/dashboard_legacy/static/media/logo_small.4204397d.svg +0 -3
  552. zenml/zen_server/dashboard_legacy/static/media/logo_white.d4b4414e.svg +0 -20
  553. zenml/zen_server/dashboard_legacy/static/media/notConnected.5e2c8ea7.svg +0 -8
  554. zenml/zen_server/dashboard_legacy/static/media/plugin-fallback.72c294e6.svg +0 -6
  555. zenml/zen_server/dashboard_legacy/static/media/share.bcd998b0.svg +0 -5
  556. zenml/zen_server/dashboard_legacy/static/media/stars.08a9b19a.svg +0 -8
  557. zenml/zen_server/deploy/terraform/__init__.py +0 -41
  558. zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
  559. zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
  560. zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
  561. zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -331
  562. zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
  563. zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
  564. zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
  565. zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -7
  566. zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
  567. zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
  568. zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
  569. zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -167
  570. zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
  571. zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -103
  572. zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
  573. zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
  574. zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
  575. zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
  576. zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -7
  577. zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
  578. zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
  579. zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
  580. zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
  581. zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -176
  582. zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -103
  583. zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
  584. zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
  585. zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
  586. zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -7
  587. zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
  588. zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
  589. zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
  590. zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -171
  591. zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -114
  592. zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -246
  593. /zenml/{new/pipelines → pipelines}/build_utils.py +0 -0
  594. /zenml/{new/steps → steps}/decorated_step.py +0 -0
  595. {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/LICENSE +0 -0
  596. {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,574 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Utilities for creating step runs."""
15
+
16
+ from datetime import datetime
17
+ from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
18
+
19
+ from zenml.client import Client
20
+ from zenml.config.step_configurations import ArtifactConfiguration, Step
21
+ from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
22
+ from zenml.enums import ExecutionStatus
23
+ from zenml.logger import get_logger
24
+ from zenml.model.utils import link_artifact_version_to_model_version
25
+ from zenml.models import (
26
+ ArtifactVersionResponse,
27
+ ModelVersionPipelineRunRequest,
28
+ ModelVersionResponse,
29
+ PipelineDeploymentResponse,
30
+ PipelineRunResponse,
31
+ PipelineRunUpdate,
32
+ StepRunRequest,
33
+ StepRunResponse,
34
+ StepRunUpdate,
35
+ )
36
+ from zenml.orchestrators import cache_utils, input_utils, utils
37
+ from zenml.stack import Stack
38
+ from zenml.utils import pagination_utils, string_utils
39
+
40
+ if TYPE_CHECKING:
41
+ from zenml.model.model import Model
42
+
43
+ logger = get_logger(__name__)
44
+
45
+
46
+ class StepRunRequestFactory:
47
+ """Helper class to create step run requests."""
48
+
49
+ def __init__(
50
+ self,
51
+ deployment: "PipelineDeploymentResponse",
52
+ pipeline_run: "PipelineRunResponse",
53
+ stack: "Stack",
54
+ ) -> None:
55
+ """Initialize the object.
56
+
57
+ Args:
58
+ deployment: The deployment for which to create step run requests.
59
+ pipeline_run: The pipeline run for which to create step run
60
+ requests.
61
+ stack: The stack on which the pipeline run is happening.
62
+ """
63
+ self.deployment = deployment
64
+ self.pipeline_run = pipeline_run
65
+ self.stack = stack
66
+
67
+ def create_request(self, invocation_id: str) -> StepRunRequest:
68
+ """Create a step run request.
69
+
70
+ This will only create a request with basic information and will not yet
71
+ compute information like the cache key and inputs. This is separated
72
+ into a different method `populate_request(...)` that might raise
73
+ exceptions while trying to compute this information.
74
+
75
+ Args:
76
+ invocation_id: The invocation ID for which to create the request.
77
+
78
+ Returns:
79
+ The step run request.
80
+ """
81
+ return StepRunRequest(
82
+ name=invocation_id,
83
+ pipeline_run_id=self.pipeline_run.id,
84
+ deployment=self.deployment.id,
85
+ status=ExecutionStatus.RUNNING,
86
+ start_time=datetime.utcnow(),
87
+ user=Client().active_user.id,
88
+ workspace=Client().active_workspace.id,
89
+ )
90
+
91
+ def populate_request(self, request: StepRunRequest) -> None:
92
+ """Populate a step run request with additional information.
93
+
94
+ Args:
95
+ request: The request to populate.
96
+ """
97
+ step = self.deployment.step_configurations[request.name]
98
+
99
+ input_artifacts, parent_step_ids = input_utils.resolve_step_inputs(
100
+ step=step,
101
+ pipeline_run=self.pipeline_run,
102
+ )
103
+ input_artifact_ids = {
104
+ input_name: artifact.id
105
+ for input_name, artifact in input_artifacts.items()
106
+ }
107
+ request.inputs = input_artifact_ids
108
+ request.parent_step_ids = parent_step_ids
109
+
110
+ cache_key = cache_utils.generate_cache_key(
111
+ step=step,
112
+ input_artifact_ids=input_artifact_ids,
113
+ artifact_store=self.stack.artifact_store,
114
+ workspace_id=Client().active_workspace.id,
115
+ )
116
+ request.cache_key = cache_key
117
+
118
+ (
119
+ docstring,
120
+ source_code,
121
+ ) = self._get_docstring_and_source_code(invocation_id=request.name)
122
+
123
+ request.docstring = docstring
124
+ request.source_code = source_code
125
+ request.code_hash = step.config.parameters.get(
126
+ CODE_HASH_PARAMETER_NAME
127
+ )
128
+
129
+ cache_enabled = utils.is_setting_enabled(
130
+ is_enabled_on_step=step.config.enable_cache,
131
+ is_enabled_on_pipeline=self.deployment.pipeline_configuration.enable_cache,
132
+ )
133
+
134
+ if cache_enabled:
135
+ if cached_step_run := cache_utils.get_cached_step_run(
136
+ cache_key=cache_key
137
+ ):
138
+ request.inputs = {
139
+ input_name: artifact.id
140
+ for input_name, artifact in cached_step_run.inputs.items()
141
+ }
142
+
143
+ request.original_step_run_id = cached_step_run.id
144
+ request.outputs = {
145
+ output_name: artifact.id
146
+ for output_name, artifact in cached_step_run.outputs.items()
147
+ }
148
+
149
+ request.status = ExecutionStatus.CACHED
150
+ request.end_time = request.start_time
151
+
152
+ def _get_docstring_and_source_code(
153
+ self, invocation_id: str
154
+ ) -> Tuple[Optional[str], Optional[str]]:
155
+ """Get the docstring and source code for the step.
156
+
157
+ Args:
158
+ invocation_id: The step invocation ID for which to get the
159
+ docstring and source code.
160
+
161
+ Returns:
162
+ The docstring and source code of the step.
163
+ """
164
+ step = self.deployment.step_configurations[invocation_id]
165
+
166
+ try:
167
+ return self._get_docstring_and_source_code_from_step_instance(
168
+ step=step
169
+ )
170
+ except ImportError:
171
+ pass
172
+
173
+ # Failed to import the step instance, this is most likely because this
174
+ # code is running on the server as part of a template execution.
175
+ # We now try to fetch the docstring/source code from a step run of the
176
+ # deployment that was used to create the template
177
+ return self._try_to_get_docstring_and_source_code_from_template(
178
+ invocation_id=invocation_id
179
+ )
180
+
181
+ @staticmethod
182
+ def _get_docstring_and_source_code_from_step_instance(
183
+ step: "Step",
184
+ ) -> Tuple[Optional[str], str]:
185
+ """Get the docstring and source code of a step.
186
+
187
+ Args:
188
+ step: The step configuration.
189
+
190
+ Returns:
191
+ The docstring and source code of a step.
192
+ """
193
+ from zenml.steps.base_step import BaseStep
194
+
195
+ step_instance = BaseStep.load_from_source(step.spec.source)
196
+
197
+ docstring = step_instance.docstring
198
+ if docstring and len(docstring) > TEXT_FIELD_MAX_LENGTH:
199
+ docstring = docstring[: (TEXT_FIELD_MAX_LENGTH - 3)] + "..."
200
+
201
+ source_code = step_instance.source_code
202
+ if source_code and len(source_code) > TEXT_FIELD_MAX_LENGTH:
203
+ source_code = source_code[: (TEXT_FIELD_MAX_LENGTH - 3)] + "..."
204
+
205
+ return docstring, source_code
206
+
207
+ def _try_to_get_docstring_and_source_code_from_template(
208
+ self, invocation_id: str
209
+ ) -> Tuple[Optional[str], Optional[str]]:
210
+ """Try to get the docstring and source code via a potential template.
211
+
212
+ Args:
213
+ invocation_id: The step invocation ID for which to get the
214
+ docstring and source code.
215
+
216
+ Returns:
217
+ The docstring and source code of the step.
218
+ """
219
+ if template_id := self.pipeline_run.template_id:
220
+ template = Client().get_run_template(template_id)
221
+ if (
222
+ deployment_id := template.source_deployment.id
223
+ if template.source_deployment
224
+ else None
225
+ ):
226
+ steps = Client().list_run_steps(
227
+ deployment_id=deployment_id,
228
+ name=invocation_id,
229
+ size=1,
230
+ hydrate=True,
231
+ )
232
+
233
+ if len(steps) > 0:
234
+ step = steps[0]
235
+ return step.docstring, step.source_code
236
+
237
+ return None, None
238
+
239
+
240
+ def find_cacheable_invocation_candidates(
241
+ deployment: "PipelineDeploymentResponse",
242
+ finished_invocations: Set[str],
243
+ ) -> Set[str]:
244
+ """Find invocations that can potentially be cached.
245
+
246
+ Args:
247
+ deployment: The pipeline deployment containing the invocations.
248
+ finished_invocations: A set of invocations that are already finished.
249
+
250
+ Returns:
251
+ The set of invocations that can potentially be cached.
252
+ """
253
+ invocations = set()
254
+ for invocation_id, step in deployment.step_configurations.items():
255
+ if invocation_id in finished_invocations:
256
+ continue
257
+
258
+ cache_enabled = utils.is_setting_enabled(
259
+ is_enabled_on_step=step.config.enable_cache,
260
+ is_enabled_on_pipeline=deployment.pipeline_configuration.enable_cache,
261
+ )
262
+
263
+ if not cache_enabled:
264
+ continue
265
+
266
+ if set(step.spec.upstream_steps) - finished_invocations:
267
+ continue
268
+
269
+ invocations.add(invocation_id)
270
+
271
+ return invocations
272
+
273
+
274
+ def create_cached_step_runs(
275
+ deployment: "PipelineDeploymentResponse",
276
+ pipeline_run: PipelineRunResponse,
277
+ stack: "Stack",
278
+ ) -> Set[str]:
279
+ """Create all cached step runs for a pipeline run.
280
+
281
+ Args:
282
+ deployment: The deployment of the pipeline run.
283
+ pipeline_run: The pipeline run for which to create the step runs.
284
+ stack: The stack on which the pipeline run is happening.
285
+
286
+ Returns:
287
+ The invocation IDs of the created step runs.
288
+ """
289
+ cached_invocations: Set[str] = set()
290
+ visited_invocations: Set[str] = set()
291
+ request_factory = StepRunRequestFactory(
292
+ deployment=deployment, pipeline_run=pipeline_run, stack=stack
293
+ )
294
+
295
+ pipeline_model_version, pipeline_run = prepare_pipeline_run_model_version(
296
+ pipeline_run=pipeline_run
297
+ )
298
+
299
+ while (
300
+ cache_candidates := find_cacheable_invocation_candidates(
301
+ deployment=deployment,
302
+ finished_invocations=cached_invocations,
303
+ )
304
+ # We've already checked these invocations and were not able to cache
305
+ # them -> no need to check them again
306
+ - visited_invocations
307
+ ):
308
+ for invocation_id in cache_candidates:
309
+ visited_invocations.add(invocation_id)
310
+
311
+ try:
312
+ step_run_request = request_factory.create_request(
313
+ invocation_id
314
+ )
315
+ request_factory.populate_request(step_run_request)
316
+ except Exception as e:
317
+ # We failed to create/populate the step run. This might be due
318
+ # to some input resolution error, or an error importing the step
319
+ # source (there might be some missing dependencies). We just
320
+ # treat the step as not cached and let the orchestrator try
321
+ # again in a potentially different environment.
322
+ logger.debug(
323
+ "Failed preparing step run `%s`: %s", invocation_id, str(e)
324
+ )
325
+ continue
326
+
327
+ if step_run_request.status != ExecutionStatus.CACHED:
328
+ # If we're not able to cache the step run, the orchestrator
329
+ # will run the step later which will create the step run
330
+ # -> We don't need to do anything here
331
+ continue
332
+
333
+ step_run = Client().zen_store.create_run_step(step_run_request)
334
+
335
+ # Refresh the pipeline run here to make sure we have the latest
336
+ # state
337
+ pipeline_run = Client().get_pipeline_run(pipeline_run.id)
338
+
339
+ step_model_version, step_run = prepare_step_run_model_version(
340
+ step_run=step_run, pipeline_run=pipeline_run
341
+ )
342
+
343
+ if model_version := step_model_version or pipeline_model_version:
344
+ link_output_artifacts_to_model_version(
345
+ artifacts=step_run.outputs,
346
+ output_configurations=step_run.config.outputs,
347
+ model_version=model_version,
348
+ )
349
+
350
+ logger.info("Using cached version of step `%s`.", invocation_id)
351
+ cached_invocations.add(invocation_id)
352
+
353
+ return cached_invocations
354
+
355
+
356
+ def get_or_create_model_version_for_pipeline_run(
357
+ model: "Model", pipeline_run: PipelineRunResponse
358
+ ) -> Tuple[ModelVersionResponse, bool]:
359
+ """Get or create a model version as part of a pipeline run.
360
+
361
+ Args:
362
+ model: The model to get or create.
363
+ pipeline_run: The pipeline run for which the model should be created.
364
+
365
+ Returns:
366
+ The model version and a boolean indicating whether it was newly created
367
+ or not.
368
+ """
369
+ # Copy the model before modifying it so we don't accidently modify
370
+ # configurations in which the model object is potentially referenced
371
+ model = model.model_copy()
372
+
373
+ if model.model_version_id:
374
+ return model._get_model_version(), False
375
+ elif model.version:
376
+ if isinstance(model.version, str):
377
+ start_time = pipeline_run.start_time or datetime.utcnow()
378
+ model.version = string_utils.format_name_template(
379
+ model.version,
380
+ date=start_time.strftime("%Y_%m_%d"),
381
+ time=start_time.strftime("%H_%M_%S_%f"),
382
+ )
383
+
384
+ return (
385
+ model._get_or_create_model_version(),
386
+ model._created_model_version,
387
+ )
388
+
389
+ # The model version should be created as part of this run
390
+ # -> We first check if it was already created as part of this run, and if
391
+ # not we do create it. If this is running in two parallel steps, we might
392
+ # run into issues that this will create two versions. Ideally, all model
393
+ # versions required for a pipeline run and its steps could be created
394
+ # server-side at run creation time before the first step starts.
395
+ if model_version := get_model_version_created_by_pipeline_run(
396
+ model_name=model.name, pipeline_run=pipeline_run
397
+ ):
398
+ return model_version, False
399
+ else:
400
+ return model._get_or_create_model_version(), True
401
+
402
+
403
+ def get_model_version_created_by_pipeline_run(
404
+ model_name: str, pipeline_run: PipelineRunResponse
405
+ ) -> Optional[ModelVersionResponse]:
406
+ """Get a model version that was created by a specific pipeline run.
407
+
408
+ This function does not refresh the pipeline run, so it will only try to
409
+ fetch the model version from existing steps if they're already part of the
410
+ response.
411
+
412
+ Args:
413
+ model_name: The model name for which to get the version.
414
+ pipeline_run: The pipeline run for which to get the version.
415
+
416
+ Returns:
417
+ A model version with the given name created by the run, or None if such
418
+ a model version does not exist.
419
+ """
420
+ if pipeline_run.config.model and pipeline_run.model_version:
421
+ if (
422
+ pipeline_run.config.model.name == model_name
423
+ and pipeline_run.config.model.version is None
424
+ ):
425
+ return pipeline_run.model_version
426
+
427
+ # We fetch a list of hydrated step runs here in order to avoid hydration
428
+ # calls for each step separately.
429
+ candidate_step_runs = pagination_utils.depaginate(
430
+ Client().list_run_steps,
431
+ pipeline_run_id=pipeline_run.id,
432
+ model=model_name,
433
+ hydrate=True,
434
+ )
435
+ for step_run in candidate_step_runs:
436
+ if step_run.config.model and step_run.model_version:
437
+ if (
438
+ step_run.config.model.name == model_name
439
+ and step_run.config.model.version is None
440
+ ):
441
+ return step_run.model_version
442
+
443
+ return None
444
+
445
+
446
+ def prepare_pipeline_run_model_version(
447
+ pipeline_run: PipelineRunResponse,
448
+ ) -> Tuple[Optional[ModelVersionResponse], PipelineRunResponse]:
449
+ """Prepare the model version for a pipeline run.
450
+
451
+ Args:
452
+ pipeline_run: The pipeline run for which to prepare the model version.
453
+
454
+ Returns:
455
+ The prepared model version and the updated pipeline run.
456
+ """
457
+ model_version = None
458
+
459
+ if pipeline_run.model_version:
460
+ model_version = pipeline_run.model_version
461
+ elif config_model := pipeline_run.config.model:
462
+ model_version, _ = get_or_create_model_version_for_pipeline_run(
463
+ model=config_model, pipeline_run=pipeline_run
464
+ )
465
+ pipeline_run = Client().zen_store.update_run(
466
+ run_id=pipeline_run.id,
467
+ run_update=PipelineRunUpdate(model_version_id=model_version.id),
468
+ )
469
+ link_pipeline_run_to_model_version(
470
+ pipeline_run=pipeline_run, model_version=model_version
471
+ )
472
+ log_model_version_dashboard_url(model_version)
473
+
474
+ return model_version, pipeline_run
475
+
476
+
477
+ def prepare_step_run_model_version(
478
+ step_run: StepRunResponse, pipeline_run: PipelineRunResponse
479
+ ) -> Tuple[Optional[ModelVersionResponse], StepRunResponse]:
480
+ """Prepare the model version for a step run.
481
+
482
+ Args:
483
+ step_run: The step run for which to prepare the model version.
484
+ pipeline_run: The pipeline run of the step.
485
+
486
+ Returns:
487
+ The prepared model version and the updated step run.
488
+ """
489
+ model_version = None
490
+
491
+ if step_run.model_version:
492
+ model_version = step_run.model_version
493
+ elif config_model := step_run.config.model:
494
+ model_version, created = get_or_create_model_version_for_pipeline_run(
495
+ model=config_model, pipeline_run=pipeline_run
496
+ )
497
+ step_run = Client().zen_store.update_run_step(
498
+ step_run_id=step_run.id,
499
+ step_run_update=StepRunUpdate(model_version_id=model_version.id),
500
+ )
501
+ link_pipeline_run_to_model_version(
502
+ pipeline_run=pipeline_run, model_version=model_version
503
+ )
504
+ if created:
505
+ log_model_version_dashboard_url(model_version)
506
+
507
+ return model_version, step_run
508
+
509
+
510
+ def log_model_version_dashboard_url(
511
+ model_version: ModelVersionResponse,
512
+ ) -> None:
513
+ """Log the dashboard URL for a model version.
514
+
515
+ If the current server is not a ZenML Pro tenant, a fallback message is
516
+ logged instead.
517
+
518
+ Args:
519
+ model_version: The model version for which to log the dashboard URL.
520
+ """
521
+ from zenml.utils.cloud_utils import try_get_model_version_url
522
+
523
+ if model_version_url_logs := try_get_model_version_url(model_version):
524
+ logger.info(model_version_url_logs)
525
+ else:
526
+ logger.info(
527
+ "Models can be viewed in the dashboard using ZenML Pro. Sign up "
528
+ "for a free trial at https://www.zenml.io/pro/"
529
+ )
530
+
531
+
532
+ def link_pipeline_run_to_model_version(
533
+ pipeline_run: PipelineRunResponse, model_version: ModelVersionResponse
534
+ ) -> None:
535
+ """Link a pipeline run to a model version.
536
+
537
+ Args:
538
+ pipeline_run: The pipeline run to link.
539
+ model_version: The model version to link.
540
+ """
541
+ client = Client()
542
+ client.zen_store.create_model_version_pipeline_run_link(
543
+ ModelVersionPipelineRunRequest(
544
+ user=client.active_user.id,
545
+ workspace=client.active_workspace.id,
546
+ pipeline_run=pipeline_run.id,
547
+ model=model_version.model.id,
548
+ model_version=model_version.id,
549
+ )
550
+ )
551
+
552
+
553
+ def link_output_artifacts_to_model_version(
554
+ artifacts: Dict[str, ArtifactVersionResponse],
555
+ output_configurations: Mapping[str, ArtifactConfiguration],
556
+ model_version: ModelVersionResponse,
557
+ ) -> None:
558
+ """Link the outputs of a step run to a model version.
559
+
560
+ Args:
561
+ artifacts: The step output artifacts.
562
+ output_configurations: The output configurations for the step.
563
+ model_version: The model version to link.
564
+ """
565
+ for output_name, output_artifact in artifacts.items():
566
+ artifact_config = None
567
+ if output_config := output_configurations.get(output_name, None):
568
+ artifact_config = output_config.artifact_config
569
+
570
+ link_artifact_version_to_model_version(
571
+ artifact_version=output_artifact,
572
+ model_version=model_version,
573
+ artifact_config=artifact_config,
574
+ )