zenml-nightly 0.68.1.dev20241106__py3-none-any.whl → 0.70.0.dev20241116__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +77 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +2 -0
  5. zenml/artifacts/external_artifact.py +2 -1
  6. zenml/artifacts/utils.py +138 -79
  7. zenml/cli/base.py +4 -4
  8. zenml/cli/model.py +1 -6
  9. zenml/cli/stack.py +1 -0
  10. zenml/client.py +29 -74
  11. zenml/config/server_config.py +17 -1
  12. zenml/constants.py +2 -7
  13. zenml/data_validators/base_data_validator.py +2 -2
  14. zenml/enums.py +20 -4
  15. zenml/exceptions.py +4 -0
  16. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
  17. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
  18. zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
  19. zenml/integrations/evidently/__init__.py +1 -1
  20. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
  21. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
  22. zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
  23. zenml/integrations/vllm/services/vllm_deployment.py +16 -7
  24. zenml/metadata/lazy_load.py +20 -7
  25. zenml/model/model.py +17 -64
  26. zenml/model/utils.py +5 -0
  27. zenml/models/__init__.py +0 -12
  28. zenml/models/v2/base/filter.py +121 -8
  29. zenml/models/v2/core/artifact_version.py +42 -7
  30. zenml/models/v2/core/model_version.py +26 -5
  31. zenml/models/v2/core/pipeline_run.py +25 -6
  32. zenml/models/v2/core/run_metadata.py +2 -217
  33. zenml/models/v2/core/step_run.py +62 -24
  34. zenml/orchestrators/base_orchestrator.py +12 -1
  35. zenml/orchestrators/input_utils.py +44 -19
  36. zenml/orchestrators/step_launcher.py +4 -3
  37. zenml/orchestrators/step_run_utils.py +19 -15
  38. zenml/orchestrators/step_runner.py +21 -13
  39. zenml/orchestrators/utils.py +45 -26
  40. zenml/stack_deployments/aws_stack_deployment.py +23 -6
  41. zenml/stack_deployments/azure_stack_deployment.py +28 -5
  42. zenml/stack_deployments/gcp_stack_deployment.py +25 -8
  43. zenml/stack_deployments/stack_deployment.py +3 -5
  44. zenml/steps/base_step.py +1 -1
  45. zenml/steps/entrypoint_function_utils.py +3 -5
  46. zenml/steps/step_context.py +3 -2
  47. zenml/steps/utils.py +13 -2
  48. zenml/utils/metadata_utils.py +335 -0
  49. zenml/zen_server/auth.py +221 -3
  50. zenml/zen_server/cache.py +208 -0
  51. zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
  52. zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
  53. zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
  54. zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
  55. zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
  56. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
  57. zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
  58. zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
  59. zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
  60. zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
  61. zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
  62. zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
  63. zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
  64. zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
  65. zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
  66. zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
  67. zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
  68. zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
  69. zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
  70. zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
  71. zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
  72. zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
  73. zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
  74. zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
  75. zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
  76. zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
  77. zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
  78. zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
  79. zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
  80. zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
  81. zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
  82. zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
  83. zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
  84. zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
  85. zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
  86. zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
  87. zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
  88. zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
  89. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
  90. zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
  92. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
  93. zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
  94. zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
  95. zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
  96. zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
  97. zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
  98. zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
  99. zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
  100. zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
  101. zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
  105. zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
  106. zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
  107. zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
  108. zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
  109. zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
  110. zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
  111. zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
  112. zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
  113. zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
  114. zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
  115. zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
  116. zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
  117. zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
  118. zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
  119. zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
  120. zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
  121. zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
  122. zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
  123. zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
  124. zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
  125. zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
  126. zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
  127. zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
  128. zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
  129. zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
  130. zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
  133. zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
  134. zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
  135. zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
  136. zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
  137. zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
  138. zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
  139. zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
  142. zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
  143. zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
  144. zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
  145. zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
  146. zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
  147. zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
  148. zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
  149. zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
  150. zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
  151. zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
  152. zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
  153. zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
  155. zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
  156. zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
  157. zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
  158. zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
  159. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
  160. zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
  161. zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
  162. zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
  163. zenml/zen_server/dashboard/index.html +7 -7
  164. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  165. zenml/zen_server/deploy/helm/README.md +2 -2
  166. zenml/zen_server/exceptions.py +2 -0
  167. zenml/zen_server/jwt.py +30 -13
  168. zenml/zen_server/rbac/endpoint_utils.py +43 -1
  169. zenml/zen_server/rbac/utils.py +0 -2
  170. zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
  171. zenml/zen_server/routers/auth_endpoints.py +134 -102
  172. zenml/zen_server/routers/logs_endpoints.py +66 -0
  173. zenml/zen_server/routers/workspaces_endpoints.py +3 -4
  174. zenml/zen_server/template_execution/utils.py +14 -16
  175. zenml/zen_server/utils.py +27 -0
  176. zenml/zen_server/zen_server_api.py +6 -3
  177. zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
  178. zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
  179. zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
  180. zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
  181. zenml/zen_stores/rest_zen_store.py +68 -64
  182. zenml/zen_stores/schemas/artifact_schemas.py +8 -1
  183. zenml/zen_stores/schemas/model_schemas.py +27 -3
  184. zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
  185. zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
  186. zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
  187. zenml/zen_stores/schemas/step_run_schemas.py +18 -10
  188. zenml/zen_stores/sql_zen_store.py +283 -219
  189. zenml/zen_stores/zen_store_interface.py +15 -42
  190. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/METADATA +2 -2
  191. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/RECORD +194 -179
  192. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
  193. zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
  194. zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
  195. zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
  196. zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
  197. zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
  198. zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
  199. zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
  200. zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
  201. zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
  202. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  203. zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
  204. zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
  205. zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
  206. zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
  207. zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
  208. zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
  209. zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
  210. zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
  211. zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
  212. zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
  213. zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
  214. zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
  215. zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
  216. zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
  217. zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
  218. zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
  219. zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
  220. zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
  221. zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
  222. zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
  223. zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
  224. zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
  225. zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
  226. zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
  227. zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
  228. zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
  229. zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
  230. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/LICENSE +0 -0
  231. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/WHEEL +0 -0
  232. {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241116.dist-info}/entry_points.txt +0 -0
zenml/client.py CHANGED
@@ -136,9 +136,7 @@ from zenml.models import (
136
136
  PipelineResponse,
137
137
  PipelineRunFilter,
138
138
  PipelineRunResponse,
139
- RunMetadataFilter,
140
139
  RunMetadataRequest,
141
- RunMetadataResponse,
142
140
  RunTemplateFilter,
143
141
  RunTemplateRequest,
144
142
  RunTemplateResponse,
@@ -190,6 +188,7 @@ from zenml.models import (
190
188
  WorkspaceResponse,
191
189
  WorkspaceUpdate,
192
190
  )
191
+ from zenml.models.v2.core.step_run import StepRunUpdate
193
192
  from zenml.services.service import ServiceConfig
194
193
  from zenml.services.service_status import ServiceState
195
194
  from zenml.services.service_type import ServiceType
@@ -3427,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
3427
3426
  Args:
3428
3427
  id_or_prefix: The id or id prefix of the deployment.
3429
3428
  """
3430
- deployment = self.get_deployment(id_or_prefix=id_or_prefix)
3429
+ deployment = self.get_deployment(
3430
+ id_or_prefix=id_or_prefix, hydrate=False
3431
+ )
3431
3432
  self.zen_store.delete_deployment(deployment_id=deployment.id)
3432
3433
 
3433
3434
  # ------------------------------ Run templates -----------------------------
@@ -3795,6 +3796,7 @@ class Client(metaclass=ClientMetaClass):
3795
3796
  templatable: Optional[bool] = None,
3796
3797
  tag: Optional[str] = None,
3797
3798
  user: Optional[Union[UUID, str]] = None,
3799
+ run_metadata: Optional[Dict[str, str]] = None,
3798
3800
  pipeline: Optional[Union[UUID, str]] = None,
3799
3801
  code_repository: Optional[Union[UUID, str]] = None,
3800
3802
  model: Optional[Union[UUID, str]] = None,
@@ -3834,6 +3836,7 @@ class Client(metaclass=ClientMetaClass):
3834
3836
  templatable: If the runs should be templatable or not.
3835
3837
  tag: Tag to filter by.
3836
3838
  user: The name/ID of the user to filter by.
3839
+ run_metadata: The run_metadata of the run to filter by.
3837
3840
  pipeline: The name/ID of the pipeline to filter by.
3838
3841
  code_repository: Filter by code repository name/ID.
3839
3842
  model: Filter by model name/ID.
@@ -3873,6 +3876,7 @@ class Client(metaclass=ClientMetaClass):
3873
3876
  tag=tag,
3874
3877
  unlisted=unlisted,
3875
3878
  user=user,
3879
+ run_metadata=run_metadata,
3876
3880
  pipeline=pipeline,
3877
3881
  code_repository=code_repository,
3878
3882
  stack=stack,
@@ -4166,6 +4170,8 @@ class Client(metaclass=ClientMetaClass):
4166
4170
  Returns:
4167
4171
  The artifact version.
4168
4172
  """
4173
+ from zenml import get_step_context
4174
+
4169
4175
  if cll := client_lazy_loader(
4170
4176
  method_name="get_artifact_version",
4171
4177
  name_id_or_prefix=name_id_or_prefix,
@@ -4173,13 +4179,26 @@ class Client(metaclass=ClientMetaClass):
4173
4179
  hydrate=hydrate,
4174
4180
  ):
4175
4181
  return cll # type: ignore[return-value]
4176
- return self._get_entity_version_by_id_or_name_or_prefix(
4182
+
4183
+ artifact = self._get_entity_version_by_id_or_name_or_prefix(
4177
4184
  get_method=self.zen_store.get_artifact_version,
4178
4185
  list_method=self.list_artifact_versions,
4179
4186
  name_id_or_prefix=name_id_or_prefix,
4180
4187
  version=version,
4181
4188
  hydrate=hydrate,
4182
4189
  )
4190
+ try:
4191
+ step_run = get_step_context().step_run
4192
+ client = Client()
4193
+ client.zen_store.update_run_step(
4194
+ step_run_id=step_run.id,
4195
+ step_run_update=StepRunUpdate(
4196
+ loaded_artifact_versions={artifact.name: artifact.id}
4197
+ ),
4198
+ )
4199
+ except RuntimeError:
4200
+ pass # Cannot link to step run if called outside a step
4201
+ return artifact
4183
4202
 
4184
4203
  def list_artifact_versions(
4185
4204
  self,
@@ -4206,6 +4225,7 @@ class Client(metaclass=ClientMetaClass):
4206
4225
  user: Optional[Union[UUID, str]] = None,
4207
4226
  model: Optional[Union[UUID, str]] = None,
4208
4227
  pipeline_run: Optional[Union[UUID, str]] = None,
4228
+ run_metadata: Optional[Dict[str, str]] = None,
4209
4229
  tag: Optional[str] = None,
4210
4230
  hydrate: bool = False,
4211
4231
  ) -> Page[ArtifactVersionResponse]:
@@ -4237,6 +4257,7 @@ class Client(metaclass=ClientMetaClass):
4237
4257
  user: Filter by user name or ID.
4238
4258
  model: Filter by model name or ID.
4239
4259
  pipeline_run: Filter by pipeline run name or ID.
4260
+ run_metadata: Filter by run metadata.
4240
4261
  hydrate: Flag deciding whether to hydrate the output model(s)
4241
4262
  by including metadata fields in the response.
4242
4263
 
@@ -4417,7 +4438,7 @@ class Client(metaclass=ClientMetaClass):
4417
4438
  resource_id: UUID,
4418
4439
  resource_type: MetadataResourceTypes,
4419
4440
  stack_component_id: Optional[UUID] = None,
4420
- ) -> List[RunMetadataResponse]:
4441
+ ) -> None:
4421
4442
  """Create run metadata.
4422
4443
 
4423
4444
  Args:
@@ -4430,7 +4451,7 @@ class Client(metaclass=ClientMetaClass):
4430
4451
  the metadata.
4431
4452
 
4432
4453
  Returns:
4433
- The created metadata, as string to model dictionary.
4454
+ None
4434
4455
  """
4435
4456
  from zenml.metadata.metadata_types import get_metadata_type
4436
4457
 
@@ -4465,74 +4486,8 @@ class Client(metaclass=ClientMetaClass):
4465
4486
  values=values,
4466
4487
  types=types,
4467
4488
  )
4468
- return self.zen_store.create_run_metadata(run_metadata)
4469
-
4470
- def list_run_metadata(
4471
- self,
4472
- sort_by: str = "created",
4473
- page: int = PAGINATION_STARTING_PAGE,
4474
- size: int = PAGE_SIZE_DEFAULT,
4475
- logical_operator: LogicalOperators = LogicalOperators.AND,
4476
- id: Optional[Union[UUID, str]] = None,
4477
- created: Optional[Union[datetime, str]] = None,
4478
- updated: Optional[Union[datetime, str]] = None,
4479
- workspace_id: Optional[UUID] = None,
4480
- user_id: Optional[UUID] = None,
4481
- resource_id: Optional[UUID] = None,
4482
- resource_type: Optional[MetadataResourceTypes] = None,
4483
- stack_component_id: Optional[UUID] = None,
4484
- key: Optional[str] = None,
4485
- value: Optional["MetadataType"] = None,
4486
- type: Optional[str] = None,
4487
- hydrate: bool = False,
4488
- ) -> Page[RunMetadataResponse]:
4489
- """List run metadata.
4490
-
4491
- Args:
4492
- sort_by: The field to sort the results by.
4493
- page: The page number to return.
4494
- size: The number of results to return per page.
4495
- logical_operator: The logical operator to use for filtering.
4496
- id: The ID of the metadata.
4497
- created: The creation time of the metadata.
4498
- updated: The last update time of the metadata.
4499
- workspace_id: The ID of the workspace the metadata belongs to.
4500
- user_id: The ID of the user that created the metadata.
4501
- resource_id: The ID of the resource the metadata belongs to.
4502
- resource_type: The type of the resource the metadata belongs to.
4503
- stack_component_id: The ID of the stack component that produced
4504
- the metadata.
4505
- key: The key of the metadata.
4506
- value: The value of the metadata.
4507
- type: The type of the metadata.
4508
- hydrate: Flag deciding whether to hydrate the output model(s)
4509
- by including metadata fields in the response.
4510
-
4511
- Returns:
4512
- The run metadata.
4513
- """
4514
- metadata_filter_model = RunMetadataFilter(
4515
- sort_by=sort_by,
4516
- page=page,
4517
- size=size,
4518
- logical_operator=logical_operator,
4519
- id=id,
4520
- created=created,
4521
- updated=updated,
4522
- workspace_id=workspace_id,
4523
- user_id=user_id,
4524
- resource_id=resource_id,
4525
- resource_type=resource_type,
4526
- stack_component_id=stack_component_id,
4527
- key=key,
4528
- value=value,
4529
- type=type,
4530
- )
4531
- metadata_filter_model.set_scope_workspace(self.active_workspace.id)
4532
- return self.zen_store.list_run_metadata(
4533
- metadata_filter_model,
4534
- hydrate=hydrate,
4535
- )
4489
+ self.zen_store.create_run_metadata(run_metadata)
4490
+ return None
4536
4491
 
4537
4492
  # -------------------------------- Secrets ---------------------------------
4538
4493
 
@@ -19,13 +19,14 @@ from secrets import token_hex
19
19
  from typing import Any, Dict, List, Optional, Union
20
20
  from uuid import UUID
21
21
 
22
- from pydantic import BaseModel, ConfigDict, Field, model_validator
22
+ from pydantic import BaseModel, ConfigDict, Field, PositiveInt, model_validator
23
23
 
24
24
  from zenml.constants import (
25
25
  DEFAULT_ZENML_JWT_TOKEN_ALGORITHM,
26
26
  DEFAULT_ZENML_JWT_TOKEN_LEEWAY,
27
27
  DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING,
28
28
  DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT,
29
+ DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME,
29
30
  DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY,
30
31
  DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE,
31
32
  DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS,
@@ -119,6 +120,8 @@ class ServerConfiguration(BaseModel):
119
120
  time of the JWT tokens issued to clients after they have
120
121
  authenticated with the ZenML server using an OAuth 2.0 device
121
122
  that has been marked as trusted.
123
+ generic_api_token_lifetime: The lifetime in seconds that generic
124
+ short-lived API tokens issued for automation purposes are valid.
122
125
  external_login_url: The login URL of an external authenticator service
123
126
  to use with the `EXTERNAL` authentication scheme.
124
127
  external_user_info_url: The user info URL of an external authenticator
@@ -230,6 +233,12 @@ class ServerConfiguration(BaseModel):
230
233
  deployment.
231
234
  max_request_body_size_in_bytes: The maximum size of the request body in
232
235
  bytes. If not specified, the default value of 256 Kb will be used.
236
+ memcache_max_capacity: The maximum number of entries that the memory
237
+ cache can hold. If not specified, the default value of 1000 will be
238
+ used.
239
+ memcache_default_expiry: The default expiry time in seconds for cache
240
+ entries. If not specified, the default value of 30 seconds will be
241
+ used.
233
242
  """
234
243
 
235
244
  deployment_type: ServerDeploymentType = ServerDeploymentType.OTHER
@@ -257,6 +266,10 @@ class ServerConfiguration(BaseModel):
257
266
  device_expiration_minutes: Optional[int] = None
258
267
  trusted_device_expiration_minutes: Optional[int] = None
259
268
 
269
+ generic_api_token_lifetime: PositiveInt = (
270
+ DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME
271
+ )
272
+
260
273
  external_login_url: Optional[str] = None
261
274
  external_user_info_url: Optional[str] = None
262
275
  external_cookie_name: Optional[str] = None
@@ -321,6 +334,9 @@ class ServerConfiguration(BaseModel):
321
334
  DEFAULT_ZENML_SERVER_MAX_REQUEST_BODY_SIZE_IN_BYTES
322
335
  )
323
336
 
337
+ memcache_max_capacity: int = 1000
338
+ memcache_default_expiry: int = 30
339
+
324
340
  _deployment_id: Optional[UUID] = None
325
341
 
326
342
  @model_validator(mode="before")
zenml/constants.py CHANGED
@@ -167,9 +167,6 @@ ENV_ZENML_SERVER = "ZENML_SERVER"
167
167
  ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
168
168
  ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS"
169
169
  ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
170
- ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
171
- "ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES"
172
- )
173
170
  ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
174
171
  ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
175
172
  ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
@@ -270,6 +267,7 @@ ZENML_API_KEY_PREFIX = "ZENKEY_"
270
267
  DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48 # 48 hours
271
268
  DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
272
269
  DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY = 1000
270
+ DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME = 60 * 60 # 1 hour
273
271
 
274
272
  DEFAULT_ZENML_SERVER_SECURE_HEADERS_HSTS = (
275
273
  "max-age=63072000; includeSubdomains"
@@ -338,6 +336,7 @@ ARTIFACTS = "/artifacts"
338
336
  ARTIFACT_VERSIONS = "/artifact_versions"
339
337
  ARTIFACT_VISUALIZATIONS = "/artifact_visualizations"
340
338
  AUTH = "/auth"
339
+ BATCH = "/batch"
341
340
  CODE_REFERENCES = "/code_references"
342
341
  CODE_REPOSITORIES = "/code_repositories"
343
342
  COMPONENT_TYPES = "/component-types"
@@ -409,10 +408,6 @@ MODEL_METADATA_YAML_FILE_NAME = "model_metadata.yaml"
409
408
 
410
409
  # orchestrator constants
411
410
  ORCHESTRATOR_DOCKER_IMAGE_KEY = "orchestrator"
412
- PIPELINE_API_TOKEN_EXPIRES_MINUTES = handle_int_env_var(
413
- ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES,
414
- default=60 * 24, # 24 hours
415
- )
416
411
 
417
412
  # Secret constants
418
413
  SECRET_VALUES = "values"
@@ -175,7 +175,7 @@ class BaseDataValidator(StackComponent):
175
175
 
176
176
  This method should be implemented by data validators that support
177
177
  running model validation checks (e.g. confusion matrix validation,
178
- performance reports, model error analyses, etc).
178
+ performance reports, model error analyzes, etc).
179
179
 
180
180
  Unlike `data_validation`, model validation checks require that a model
181
181
  be present as an active component during the validation process.
@@ -184,7 +184,7 @@ class BaseDataValidator(StackComponent):
184
184
  accommodate different categories of data validation tests, e.g.:
185
185
 
186
186
  * single dataset tests: confusion matrix validation,
187
- performance reports, model error analyses, etc
187
+ performance reports, model error analyzes, etc
188
188
  * model comparison tests: tests that identify changes in a model
189
189
  behavior by comparing how it performs on two different datasets.
190
190
 
zenml/enums.py CHANGED
@@ -34,15 +34,23 @@ class ArtifactType(StrEnum):
34
34
  class StepRunInputArtifactType(StrEnum):
35
35
  """All possible types of a step run input artifact."""
36
36
 
37
- DEFAULT = "default" # input argument that is the output of a previous step
37
+ STEP_OUTPUT = (
38
+ "step_output" # input argument that is the output of a previous step
39
+ )
38
40
  MANUAL = "manual" # manually loaded via `zenml.load_artifact()`
41
+ EXTERNAL = "external" # loaded via `ExternalArtifact(value=...)`
42
+ LAZY_LOADED = "lazy" # loaded via various lazy methods
39
43
 
40
44
 
41
- class StepRunOutputArtifactType(StrEnum):
42
- """All possible types of a step run output artifact."""
45
+ class ArtifactSaveType(StrEnum):
46
+ """All possible method types of how artifact versions can be saved."""
43
47
 
44
- DEFAULT = "default" # output of the current step
48
+ STEP_OUTPUT = "step_output" # output of the current step
45
49
  MANUAL = "manual" # manually saved via `zenml.save_artifact()`
50
+ PREEXISTING = "preexisting" # register via `zenml.register_artifact()`
51
+ EXTERNAL = (
52
+ "external" # saved via `zenml.ExternalArtifact.upload_by_value()`
53
+ )
46
54
 
47
55
 
48
56
  class VisualizationType(StrEnum):
@@ -237,6 +245,13 @@ class OAuthDeviceStatus(StrEnum):
237
245
  LOCKED = "locked"
238
246
 
239
247
 
248
+ class APITokenType(StrEnum):
249
+ """The API token type."""
250
+
251
+ GENERIC = "generic"
252
+ WORKLOAD = "workload"
253
+
254
+
240
255
  class GenericFilterOps(StrEnum):
241
256
  """Ops for all filters for string values on list methods."""
242
257
 
@@ -245,6 +260,7 @@ class GenericFilterOps(StrEnum):
245
260
  CONTAINS = "contains"
246
261
  STARTSWITH = "startswith"
247
262
  ENDSWITH = "endswith"
263
+ ONEOF = "oneof"
248
264
  GTE = "gte"
249
265
  GT = "gt"
250
266
  LTE = "lte"
zenml/exceptions.py CHANGED
@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
168
168
  """Raised when trying to register an entity that already exists."""
169
169
 
170
170
 
171
+ class EntityCreationError(ZenMLBaseException, RuntimeError):
172
+ """Raised when failing to create an entity."""
173
+
174
+
171
175
  class ActionExistsError(EntityExistsError):
172
176
  """Raised when registering an action with a name that already exists."""
173
177
 
@@ -478,7 +478,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
478
478
  )
479
479
 
480
480
  # Yield metadata based on the generated execution object
481
- yield from self.compute_metadata(execution=execution)
481
+ yield from self.compute_metadata(
482
+ execution=execution, settings=settings
483
+ )
482
484
 
483
485
  # mainly for testing purposes, we wait for the pipeline to finish
484
486
  if settings.synchronous:
@@ -518,19 +520,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
518
520
  "pipeline_execution_arn": pipeline_execution_arn,
519
521
  }
520
522
 
521
- aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
522
-
523
- region_name, _, _ = dissect_pipeline_execution_arn(
524
- pipeline_execution_arn=pipeline_execution_arn
525
- )
526
-
527
- orchestrator_logs_url = (
528
- f"https://{region_name}.console.aws.amazon.com/"
529
- f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
530
- f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
531
- f"$3Dpipelines-{aws_run_id}-"
532
- )
533
- run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
534
523
  return run_metadata
535
524
 
536
525
  def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
@@ -566,7 +555,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
566
555
 
567
556
  # Fetch the status of the _PipelineExecution
568
557
  if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
569
- run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID].value
558
+ run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
570
559
  elif run.orchestrator_run_id is not None:
571
560
  run_id = run.orchestrator_run_id
572
561
  else:
@@ -590,12 +579,15 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
590
579
  raise ValueError("Unknown status for the pipeline execution.")
591
580
 
592
581
  def compute_metadata(
593
- self, execution: Any
582
+ self,
583
+ execution: Any,
584
+ settings: SagemakerOrchestratorSettings,
594
585
  ) -> Iterator[Dict[str, MetadataType]]:
595
586
  """Generate run metadata based on the generated Sagemaker Execution.
596
587
 
597
588
  Args:
598
589
  execution: The corresponding _PipelineExecution object.
590
+ settings: The Sagemaker orchestrator settings.
599
591
 
600
592
  Yields:
601
593
  A dictionary of metadata related to the pipeline run.
@@ -612,7 +604,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
612
604
  metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_url)
613
605
 
614
606
  # URL to the corresponding CloudWatch page
615
- if logs_url := self._compute_orchestrator_logs_url(execution):
607
+ if logs_url := self._compute_orchestrator_logs_url(
608
+ execution, settings
609
+ ):
616
610
  metadata[METADATA_ORCHESTRATOR_LOGS_URL] = Uri(logs_url)
617
611
 
618
612
  yield metadata
@@ -656,11 +650,13 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
656
650
  @staticmethod
657
651
  def _compute_orchestrator_logs_url(
658
652
  pipeline_execution: Any,
653
+ settings: SagemakerOrchestratorSettings,
659
654
  ) -> Optional[str]:
660
655
  """Generate the CloudWatch URL upon pipeline execution.
661
656
 
662
657
  Args:
663
658
  pipeline_execution: The corresponding _PipelineExecution object.
659
+ settings: The Sagemaker orchestrator settings.
664
660
 
665
661
  Returns:
666
662
  the URL querying the pipeline logs in CloudWatch on AWS.
@@ -670,10 +666,16 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
670
666
  pipeline_execution.arn
671
667
  )
672
668
 
669
+ use_training_jobs = True
670
+ if settings.use_training_step is not None:
671
+ use_training_jobs = settings.use_training_step
672
+
673
+ job_type = "Training" if use_training_jobs else "Processing"
674
+
673
675
  return (
674
676
  f"https://{region_name}.console.aws.amazon.com/"
675
677
  f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
676
- f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
678
+ f"/$252Faws$252Fsagemaker$252F{job_type}Jobs$3FlogStreamNameFilter"
677
679
  f"$3Dpipelines-{execution_id}-"
678
680
  )
679
681
  except Exception as e:
@@ -482,7 +482,7 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
482
482
 
483
483
  # Fetch the status of the PipelineJob
484
484
  if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
485
- run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID].value
485
+ run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
486
486
  elif run.orchestrator_run_id is not None:
487
487
  run_id = run.orchestrator_run_id
488
488
  else:
@@ -430,7 +430,7 @@ class DeepchecksDataValidator(BaseDataValidator):
430
430
  """Run one or more Deepchecks model validation checks.
431
431
 
432
432
  Call this method to perform model validation checks (e.g. confusion
433
- matrix validation, performance reports, model error analyses, etc).
433
+ matrix validation, performance reports, model error analyzes, etc).
434
434
  A second dataset is required for model performance comparison tests
435
435
  (i.e. tests that identify changes in a model behavior by comparing how
436
436
  it performs on two different datasets).
@@ -33,7 +33,7 @@ from zenml.stack import Flavor
33
33
 
34
34
  # Fix numba errors in Docker and suppress logs and deprecation warning spam
35
35
  try:
36
- from numba.core.errors import ( # type: ignore[import-not-found]
36
+ from numba.core.errors import (
37
37
  NumbaDeprecationWarning,
38
38
  NumbaPendingDeprecationWarning,
39
39
  )
@@ -835,7 +835,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
835
835
 
836
836
  # Fetch the status of the PipelineJob
837
837
  if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
838
- run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID].value
838
+ run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
839
839
  elif run.orchestrator_run_id is not None:
840
840
  run_id = run.orchestrator_run_id
841
841
  else:
@@ -390,10 +390,13 @@ class LightningOrchestrator(WheeledOrchestrator):
390
390
  studio.run(
391
391
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
392
392
  )
393
- studio.upload_file(
394
- env_file_path,
395
- remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
393
+ studio.upload_file(env_file_path)
394
+ time.sleep(6)
395
+ studio.run(
396
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
396
397
  )
398
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
399
+
397
400
  studio.run("pip install uv")
398
401
  logger.info(
399
402
  f"Installing requirements: {pipeline_requirements_to_string}"
@@ -481,16 +484,25 @@ class LightningOrchestrator(WheeledOrchestrator):
481
484
  code_path,
482
485
  remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
483
486
  )
487
+ time.sleep(6)
484
488
  studio.run(
485
489
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
486
490
  )
487
491
  logger.info(
488
492
  "Uploading wheel package and installing dependencies on main studio"
489
493
  )
490
- studio.upload_file(
491
- env_file_path,
492
- remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
494
+ # for some reason uploading file directly to /teamspace/studios/this_studio/.lightning_studio/.studiorc
495
+ # doesn't work, while other file names can be uploaded just fine.
496
+ # below is a workaround to copy the file to the correct location.
497
+ # we first upload it under a different name and then copy it to the
498
+ # right location.
499
+ studio.upload_file(env_file_path)
500
+ time.sleep(6)
501
+ studio.run(
502
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
493
503
  )
504
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
505
+
494
506
  studio.run("pip install uv")
495
507
  studio.run(f"uv pip install {requirements}")
496
508
  studio.run("pip install zenml")
@@ -557,13 +569,21 @@ class LightningOrchestrator(WheeledOrchestrator):
557
569
  studio.run(
558
570
  f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
559
571
  )
560
- studio.upload_file(code_path, remote_path=f"/zenml_codes/{filename}")
572
+ studio.upload_file(
573
+ code_path,
574
+ remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
575
+ )
576
+ time.sleep(6)
561
577
  studio.run(
562
578
  f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
563
579
  )
564
- studio.upload_file(
565
- env_file_path, remote_path=".lightning_studio/.studiorc"
580
+ studio.upload_file(env_file_path)
581
+ time.sleep(6)
582
+ studio.run(
583
+ f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
566
584
  )
585
+ studio.run(f"rm {env_file_path.split('/')[-1]}")
586
+
567
587
  studio.run("pip install uv")
568
588
  studio.run(f"uv pip install {details['requirements']}")
569
589
  studio.run("pip install zenml")