zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
zenml/steps/base_step.py CHANGED
@@ -22,6 +22,7 @@ from typing import (
22
22
  TYPE_CHECKING,
23
23
  Any,
24
24
  Dict,
25
+ List,
25
26
  Mapping,
26
27
  Optional,
27
28
  Sequence,
@@ -38,7 +39,11 @@ from pydantic import BaseModel, ConfigDict, ValidationError
38
39
  from zenml.client_lazy_loader import ClientLazyLoader
39
40
  from zenml.config.retry_config import StepRetryConfig
40
41
  from zenml.config.source import Source
41
- from zenml.constants import STEP_SOURCE_PARAMETER_NAME
42
+ from zenml.constants import (
43
+ ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK,
44
+ STEP_SOURCE_PARAMETER_NAME,
45
+ handle_bool_env_var,
46
+ )
42
47
  from zenml.exceptions import MissingStepParameterError, StepInterfaceError
43
48
  from zenml.logger import get_logger
44
49
  from zenml.materializers.base_materializer import BaseMaterializer
@@ -51,15 +56,16 @@ from zenml.steps.entrypoint_function_utils import (
51
56
  )
52
57
  from zenml.steps.utils import (
53
58
  resolve_type_annotation,
59
+ run_as_single_step_pipeline,
54
60
  )
55
61
  from zenml.utils import (
56
62
  dict_utils,
63
+ materializer_utils,
57
64
  notebook_utils,
58
65
  pydantic_utils,
59
66
  settings_utils,
60
67
  source_code_utils,
61
68
  source_utils,
62
- typing_utils,
63
69
  )
64
70
 
65
71
  if TYPE_CHECKING:
@@ -586,9 +592,16 @@ class BaseStep(metaclass=BaseStepMeta):
586
592
  from zenml.new.pipelines.pipeline import Pipeline
587
593
 
588
594
  if not Pipeline.ACTIVE_PIPELINE:
589
- # The step is being called outside the context of a pipeline,
590
- # we simply call the entrypoint
591
- return self.call_entrypoint(*args, **kwargs)
595
+ # The step is being called outside the context of a pipeline, either
596
+ # run the step function or run it as a single step pipeline on the
597
+ # active stack
598
+ run_without_stack = handle_bool_env_var(
599
+ ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK, default=False
600
+ )
601
+ if run_without_stack:
602
+ return self.call_entrypoint(*args, **kwargs)
603
+ else:
604
+ return run_as_single_step_pipeline(self, *args, **kwargs)
592
605
 
593
606
  (
594
607
  input_artifacts,
@@ -1098,6 +1111,11 @@ To avoid this consider setting step parameters only in one place (config or code
1098
1111
  this step.
1099
1112
  client_lazy_loaders: The client lazy loaders of this step.
1100
1113
 
1114
+ Raises:
1115
+ StepInterfaceError: If explicit materializers were specified for an
1116
+ output but they do not work for the data type(s) defined by
1117
+ the type annotation.
1118
+
1101
1119
  Returns:
1102
1120
  The finalized step configuration.
1103
1121
  """
@@ -1119,9 +1137,45 @@ To avoid this consider setting step parameters only in one place (config or code
1119
1137
  output_name, PartialArtifactConfiguration()
1120
1138
  )
1121
1139
 
1122
- from zenml.steps.utils import get_args
1140
+ if output.materializer_source:
1141
+ # The materializer source was configured by the user. We
1142
+ # validate that their configured materializer supports the
1143
+ # output type. If the output annotation is a Union, we check
1144
+ # that at least one of the specified materializers works with at
1145
+ # least one of the types in the Union. If that's not the case,
1146
+ # it would be a guaranteed failure at runtime and we fail early
1147
+ # here.
1148
+ if output_annotation.resolved_annotation is Any:
1149
+ continue
1123
1150
 
1124
- if not output.materializer_source:
1151
+ materializer_classes: List[Type["BaseMaterializer"]] = [
1152
+ source_utils.load(materializer_source)
1153
+ for materializer_source in output.materializer_source
1154
+ ]
1155
+
1156
+ for data_type in output_annotation.get_output_types():
1157
+ try:
1158
+ materializer_utils.select_materializer(
1159
+ data_type=data_type,
1160
+ materializer_classes=materializer_classes,
1161
+ )
1162
+ break
1163
+ except RuntimeError:
1164
+ pass
1165
+ else:
1166
+ materializer_strings = [
1167
+ materializer_source.import_path
1168
+ for materializer_source in output.materializer_source
1169
+ ]
1170
+ raise StepInterfaceError(
1171
+ "Invalid materializers specified for output "
1172
+ f"{output_name} of step {self.name}. None of the "
1173
+ f"materializers ({materializer_strings}) are "
1174
+ "able to save or load data of the type that is defined "
1175
+ "for the output "
1176
+ f"({output_annotation.resolved_annotation})."
1177
+ )
1178
+ else:
1125
1179
  if output_annotation.resolved_annotation is Any:
1126
1180
  outputs[output_name]["materializer_source"] = ()
1127
1181
  outputs[output_name]["default_materializer_source"] = (
@@ -1131,26 +1185,9 @@ To avoid this consider setting step parameters only in one place (config or code
1131
1185
  )
1132
1186
  continue
1133
1187
 
1134
- if typing_utils.is_union(
1135
- typing_utils.get_origin(
1136
- output_annotation.resolved_annotation
1137
- )
1138
- or output_annotation.resolved_annotation
1139
- ):
1140
- output_types = tuple(
1141
- type(None)
1142
- if typing_utils.is_none_type(output_type)
1143
- else output_type
1144
- for output_type in get_args(
1145
- output_annotation.resolved_annotation
1146
- )
1147
- )
1148
- else:
1149
- output_types = (output_annotation.resolved_annotation,)
1150
-
1151
1188
  materializer_sources = []
1152
1189
 
1153
- for output_type in output_types:
1190
+ for output_type in output_annotation.get_output_types():
1154
1191
  materializer_class = materializer_registry[output_type]
1155
1192
  materializer_sources.append(
1156
1193
  source_utils.resolve(materializer_class)
zenml/steps/utils.py CHANGED
@@ -18,7 +18,7 @@ import ast
18
18
  import contextlib
19
19
  import inspect
20
20
  import textwrap
21
- from typing import Any, Callable, Dict, Optional, Tuple, Union
21
+ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union
22
22
  from uuid import UUID
23
23
 
24
24
  from pydantic import BaseModel
@@ -26,12 +26,17 @@ from typing_extensions import Annotated
26
26
 
27
27
  from zenml.artifacts.artifact_config import ArtifactConfig
28
28
  from zenml.client import Client
29
- from zenml.enums import MetadataResourceTypes
29
+ from zenml.enums import ExecutionStatus, MetadataResourceTypes
30
+ from zenml.exceptions import StepInterfaceError
30
31
  from zenml.logger import get_logger
31
32
  from zenml.metadata.metadata_types import MetadataType
32
33
  from zenml.new.steps.step_context import get_step_context
33
34
  from zenml.steps.step_output import Output
34
- from zenml.utils import source_code_utils, typing_utils
35
+ from zenml.utils import settings_utils, source_code_utils, typing_utils
36
+
37
+ if TYPE_CHECKING:
38
+ from zenml.steps import BaseStep
39
+
35
40
 
36
41
  logger = get_logger(__name__)
37
42
 
@@ -45,6 +50,28 @@ class OutputSignature(BaseModel):
45
50
  artifact_config: Optional[ArtifactConfig] = None
46
51
  has_custom_name: bool = False
47
52
 
53
+ def get_output_types(self) -> Tuple[Any, ...]:
54
+ """Get all output types that match the type annotation.
55
+
56
+ Returns:
57
+ All output types that match the type annotation.
58
+ """
59
+ if self.resolved_annotation is Any:
60
+ return ()
61
+
62
+ if typing_utils.is_union(
63
+ typing_utils.get_origin(self.resolved_annotation)
64
+ or self.resolved_annotation
65
+ ):
66
+ return tuple(
67
+ type(None)
68
+ if typing_utils.is_none_type(output_type)
69
+ else output_type
70
+ for output_type in get_args(self.resolved_annotation)
71
+ )
72
+ else:
73
+ return (self.resolved_annotation,)
74
+
48
75
 
49
76
  def get_args(obj: Any) -> Tuple[Any, ...]:
50
77
  """Get arguments of a type annotation.
@@ -464,3 +491,88 @@ def log_step_metadata(
464
491
  resource_id=step_run_id,
465
492
  resource_type=MetadataResourceTypes.STEP_RUN,
466
493
  )
494
+
495
+
496
+ def run_as_single_step_pipeline(
497
+ __step: "BaseStep", *args: Any, **kwargs: Any
498
+ ) -> Any:
499
+ """Runs the step as a single step pipeline.
500
+
501
+ - All inputs that are not JSON serializable will be uploaded to the
502
+ artifact store before the pipeline is being executed.
503
+ - All output artifacts of the step will be loaded using the materializer
504
+ that was used to store them.
505
+
506
+ Args:
507
+ *args: Entrypoint function arguments.
508
+ **kwargs: Entrypoint function keyword arguments.
509
+
510
+ Raises:
511
+ RuntimeError: If the step execution failed.
512
+ StepInterfaceError: If the arguments to the entrypoint function are
513
+ invalid.
514
+
515
+ Returns:
516
+ The output of the step entrypoint function.
517
+ """
518
+ from zenml import ExternalArtifact, pipeline
519
+ from zenml.config.base_settings import BaseSettings
520
+ from zenml.new.pipelines.run_utils import (
521
+ wait_for_pipeline_run_to_finish,
522
+ )
523
+
524
+ logger.info(
525
+ "Running single step pipeline to execute step `%s`", __step.name
526
+ )
527
+
528
+ try:
529
+ validated_arguments = (
530
+ inspect.signature(__step.entrypoint)
531
+ .bind(*args, **kwargs)
532
+ .arguments
533
+ )
534
+ except TypeError as e:
535
+ raise StepInterfaceError(
536
+ "Invalid step function entrypoint arguments. Check out the "
537
+ "error above for more details."
538
+ ) from e
539
+
540
+ inputs: Dict[str, Any] = {}
541
+ for key, value in validated_arguments.items():
542
+ try:
543
+ __step.entrypoint_definition.validate_input(key=key, value=value)
544
+ inputs[key] = value
545
+ except Exception:
546
+ inputs[key] = ExternalArtifact(value=value)
547
+
548
+ orchestrator = Client().active_stack.orchestrator
549
+
550
+ pipeline_settings: Any = {}
551
+ if "synchronous" in orchestrator.config.model_fields:
552
+ # Make sure the orchestrator runs sync so we stream the logs
553
+ key = settings_utils.get_stack_component_setting_key(orchestrator)
554
+ pipeline_settings[key] = BaseSettings(synchronous=True)
555
+
556
+ @pipeline(name=__step.name, enable_cache=False, settings=pipeline_settings)
557
+ def single_step_pipeline() -> None:
558
+ __step(**inputs)
559
+
560
+ run = single_step_pipeline.with_options(unlisted=True)()
561
+ run = wait_for_pipeline_run_to_finish(run.id)
562
+
563
+ if run.status != ExecutionStatus.COMPLETED:
564
+ raise RuntimeError("Failed to execute step %s.", __step.name)
565
+
566
+ # 4. Load output artifacts
567
+ step_run = next(iter(run.steps.values()))
568
+ outputs = [
569
+ step_run.outputs[output_name].load()
570
+ for output_name in step_run.config.outputs.keys()
571
+ ]
572
+
573
+ if len(outputs) == 0:
574
+ return None
575
+ elif len(outputs) == 1:
576
+ return outputs[0]
577
+ else:
578
+ return tuple(outputs)
@@ -20,21 +20,21 @@ from zenml.utils.dashboard_utils import get_model_version_url
20
20
  logger = get_logger(__name__)
21
21
 
22
22
 
23
- def is_cloud_model_version(model_version: ModelVersionResponse) -> bool:
24
- """Check if a model version is from a ZenML Pro server.
23
+ def try_get_model_version_url(model_version: ModelVersionResponse) -> str:
24
+ """Check if a model version is from a ZenML Pro server and return its' URL.
25
25
 
26
26
  Args:
27
27
  model_version: The model version to check.
28
28
 
29
29
  Returns:
30
- True if the model version is from a ZenML Pro server, else False.
30
+ URL if the model version is from a ZenML Pro server, else empty string.
31
31
  """
32
32
  model_version_url = get_model_version_url(model_version.id)
33
33
  if model_version_url:
34
- logger.info(
35
- f"Dashboard URL for Model Version with name {model_version.name} "
36
- f": {model_version_url}"
34
+ return (
35
+ "Dashboard URL for created Model Version "
36
+ f"`{model_version.model.name}::{model_version.name}`:\n"
37
+ + model_version_url
37
38
  )
38
- return True
39
39
  else:
40
- return False
40
+ return ""
zenml/utils/code_utils.py CHANGED
@@ -16,6 +16,7 @@
16
16
  import hashlib
17
17
  import os
18
18
  import shutil
19
+ import sys
19
20
  import tempfile
20
21
  from pathlib import Path
21
22
  from typing import IO, TYPE_CHECKING, Dict, Optional
@@ -23,12 +24,14 @@ from typing import IO, TYPE_CHECKING, Dict, Optional
23
24
  from zenml.client import Client
24
25
  from zenml.io import fileio
25
26
  from zenml.logger import get_logger
26
- from zenml.utils import string_utils
27
+ from zenml.utils import source_utils, string_utils
27
28
  from zenml.utils.archivable import Archivable
28
29
 
29
30
  if TYPE_CHECKING:
30
31
  from git.repo.base import Repo
31
32
 
33
+ from zenml.artifact_stores import BaseArtifactStore
34
+
32
35
 
33
36
  logger = get_logger(__name__)
34
37
 
@@ -168,6 +171,30 @@ class CodeArchive(Archivable):
168
171
  )
169
172
 
170
173
 
174
+ def compute_file_hash(file: IO[bytes]) -> str:
175
+ """Compute a hash of the content of a file.
176
+
177
+ This function will not seek the file before or after the hash computation.
178
+ This means that the content will be computed based on the current cursor
179
+ until the end of the file.
180
+
181
+ Args:
182
+ file: The file for which to compute the hash.
183
+
184
+ Returns:
185
+ A hash of the file content.
186
+ """
187
+ hash_ = hashlib.sha1() # nosec
188
+
189
+ while True:
190
+ data = file.read(64 * 1024)
191
+ if not data:
192
+ break
193
+ hash_.update(data)
194
+
195
+ return hash_.hexdigest()
196
+
197
+
171
198
  def upload_code_if_necessary(code_archive: CodeArchive) -> str:
172
199
  """Upload code to the artifact store if necessary.
173
200
 
@@ -179,7 +206,7 @@ def upload_code_if_necessary(code_archive: CodeArchive) -> str:
179
206
  code_archive: The code archive to upload.
180
207
 
181
208
  Returns:
182
- The path where to archived code is uploaded.
209
+ The path where the archived code is uploaded.
183
210
  """
184
211
  artifact_store = Client().active_stack.artifact_store
185
212
 
@@ -187,36 +214,27 @@ def upload_code_if_necessary(code_archive: CodeArchive) -> str:
187
214
  mode="w+b", delete=False, suffix=".tar.gz"
188
215
  ) as f:
189
216
  code_archive.write_archive(f)
190
-
191
- hash_ = hashlib.sha1() # nosec
192
-
193
- while True:
194
- data = f.read(64 * 1024)
195
- if not data:
196
- break
197
- hash_.update(data)
198
-
199
- filename = f"{hash_.hexdigest()}.tar.gz"
200
- upload_dir = os.path.join(artifact_store.path, "code_uploads")
201
- fileio.makedirs(upload_dir)
202
- upload_path = os.path.join(upload_dir, filename)
203
-
204
- if not fileio.exists(upload_path):
205
- archive_size = string_utils.get_human_readable_filesize(
206
- os.path.getsize(f.name)
207
- )
208
- logger.info(
209
- "Uploading code to `%s` (Size: %s).", upload_path, archive_size
210
- )
211
- fileio.copy(f.name, upload_path)
212
- logger.info("Code upload finished.")
213
- else:
214
- logger.info(
215
- "Code already exists in artifact store, skipping upload."
216
- )
217
-
218
- if os.path.exists(f.name):
219
- os.remove(f.name)
217
+ archive_path = f.name
218
+ archive_hash = compute_file_hash(f)
219
+
220
+ upload_dir = os.path.join(artifact_store.path, "code_uploads")
221
+ fileio.makedirs(upload_dir)
222
+ upload_path = os.path.join(upload_dir, f"{archive_hash}.tar.gz")
223
+
224
+ if not fileio.exists(upload_path):
225
+ archive_size = string_utils.get_human_readable_filesize(
226
+ os.path.getsize(archive_path)
227
+ )
228
+ logger.info(
229
+ "Uploading code to `%s` (Size: %s).", upload_path, archive_size
230
+ )
231
+ fileio.copy(archive_path, upload_path)
232
+ logger.info("Code upload finished.")
233
+ else:
234
+ logger.info("Code already exists in artifact store, skipping upload.")
235
+
236
+ if os.path.exists(archive_path):
237
+ os.remove(archive_path)
220
238
 
221
239
  return upload_path
222
240
 
@@ -242,3 +260,83 @@ def download_and_extract_code(code_path: str, extract_dir: str) -> None:
242
260
 
243
261
  shutil.unpack_archive(filename=download_path, extract_dir=extract_dir)
244
262
  os.remove(download_path)
263
+
264
+
265
+ def download_code_from_artifact_store(code_path: str) -> None:
266
+ """Download code from the artifact store.
267
+
268
+ Args:
269
+ code_path: Path where the code is stored.
270
+ """
271
+ logger.info("Downloading code from artifact store path `%s`.", code_path)
272
+
273
+ # Do not remove this line, we need to instantiate the artifact store to
274
+ # register the filesystem needed for the file download
275
+ _ = Client().active_stack.artifact_store
276
+
277
+ extract_dir = os.path.abspath("code")
278
+ os.makedirs(extract_dir)
279
+
280
+ download_and_extract_code(code_path=code_path, extract_dir=extract_dir)
281
+
282
+ source_utils.set_custom_source_root(extract_dir)
283
+ sys.path.insert(0, extract_dir)
284
+ os.chdir(extract_dir)
285
+
286
+
287
+ def _get_notebook_upload_dir(artifact_store: "BaseArtifactStore") -> str:
288
+ """Get the upload directory for code extracted from notebook cells.
289
+
290
+ Args:
291
+ artifact_store: The artifact store in which the directory should be.
292
+
293
+ Returns:
294
+ The upload directory for code extracted from notebook cells.
295
+ """
296
+ return os.path.join(artifact_store.path, "notebook_code")
297
+
298
+
299
+ def upload_notebook_code(
300
+ artifact_store: "BaseArtifactStore", cell_code: str, file_name: str
301
+ ) -> None:
302
+ """Upload code extracted from a notebook cell.
303
+
304
+ Args:
305
+ artifact_store: The artifact store in which to upload the code.
306
+ cell_code: The notebook cell code.
307
+ file_name: The filename to use for storing the cell code.
308
+ """
309
+ upload_dir = _get_notebook_upload_dir(artifact_store=artifact_store)
310
+ fileio.makedirs(upload_dir)
311
+ upload_path = os.path.join(upload_dir, file_name)
312
+
313
+ if not fileio.exists(upload_path):
314
+ with fileio.open(upload_path, "wb") as f:
315
+ f.write(cell_code.encode())
316
+
317
+ logger.info("Uploaded notebook cell code to %s.", upload_path)
318
+
319
+
320
+ def download_notebook_code(
321
+ artifact_store: "BaseArtifactStore", file_name: str, download_path: str
322
+ ) -> None:
323
+ """Download code extracted from a notebook cell.
324
+
325
+ Args:
326
+ artifact_store: The artifact store from which to download the code.
327
+ file_name: The name of the code file.
328
+ download_path: The local path where the file should be downloaded to.
329
+
330
+ Raises:
331
+ FileNotFoundError: If no file with the given filename exists in this
332
+ artifact store.
333
+ """
334
+ code_dir = _get_notebook_upload_dir(artifact_store=artifact_store)
335
+ code_path = os.path.join(code_dir, file_name)
336
+
337
+ if not fileio.exists(code_path):
338
+ raise FileNotFoundError(
339
+ f"Notebook code at path {code_path} not found."
340
+ )
341
+
342
+ fileio.copy(code_path, download_path)
@@ -34,12 +34,12 @@ from zenml.utils.function_utils import _cli_wrapped_function
34
34
  import sys
35
35
  sys.path.append(r"{func_path}")
36
36
 
37
- from {func_module} import {func_name} as func_to_wrap
37
+ from {func_module} import {func_name} as step_function
38
38
 
39
- if entrypoint:=getattr(func_to_wrap, "entrypoint", None):
40
- func = _cli_wrapped_function(entrypoint)
39
+ if unwrapped_entrypoint:=getattr(step_function, "unwrapped_entrypoint", None):
40
+ func = _cli_wrapped_function(unwrapped_entrypoint)
41
41
  else:
42
- func = _cli_wrapped_function(func_to_wrap)
42
+ func = _cli_wrapped_function(step_function.entrypoint)
43
43
  """
44
44
  _CLI_WRAPPED_MAINS = {
45
45
  "accelerate": """
@@ -204,13 +204,13 @@ def _cli_wrapped_function(func: F) -> F:
204
204
 
205
205
  @contextmanager
206
206
  def create_cli_wrapped_script(
207
- func: F, flavour: str = "accelerate"
207
+ func: F, flavor: str = "accelerate"
208
208
  ) -> Iterator[Tuple[Path, Path]]:
209
209
  """Create a script with the CLI-wrapped function.
210
210
 
211
211
  Args:
212
212
  func: The function to use.
213
- flavour: The flavour to use.
213
+ flavor: The flavor to use.
214
214
 
215
215
  Yields:
216
216
  The paths of the script and the output.
@@ -241,7 +241,7 @@ def create_cli_wrapped_script(
241
241
  func_module=clean_module_name,
242
242
  func_name=func.__name__,
243
243
  )
244
- script += _CLI_WRAPPED_MAINS[flavour].format(
244
+ script += _CLI_WRAPPED_MAINS[flavor].format(
245
245
  output_file=str(output_path.absolute())
246
246
  )
247
247
  f.write(script)
@@ -13,6 +13,7 @@
13
13
  # permissions and limitations under the License.
14
14
  """Notebook utilities."""
15
15
 
16
+ import hashlib
16
17
  from typing import Any, Callable, Optional, TypeVar, Union
17
18
 
18
19
  from zenml.environment import Environment
@@ -120,3 +121,16 @@ def warn_about_notebook_cell_magic_commands(cell_code: str) -> None:
120
121
  "of these lines contain Jupyter notebook magic commands, "
121
122
  "remove them and try again."
122
123
  )
124
+
125
+
126
+ def compute_cell_replacement_module_name(cell_code: str) -> str:
127
+ """Compute the replacement module name for a given cell code.
128
+
129
+ Args:
130
+ cell_code: The code of the notebook cell.
131
+
132
+ Returns:
133
+ The replacement module name.
134
+ """
135
+ code_hash = hashlib.sha1(cell_code.encode()).hexdigest() # nosec
136
+ return f"extracted_notebook_code_{code_hash}"
@@ -38,7 +38,6 @@ from zenml.constants import (
38
38
  ENV_ZENML_CONFIG_PATH,
39
39
  ENV_ZENML_ENABLE_REPO_INIT_WARNINGS,
40
40
  ENV_ZENML_LOGGING_COLORS_DISABLED,
41
- ENV_ZENML_REQUIRES_CODE_DOWNLOAD,
42
41
  handle_bool_env_var,
43
42
  )
44
43
  from zenml.enums import OperatingSystemType
@@ -55,10 +54,7 @@ if TYPE_CHECKING:
55
54
  logger = get_logger(__name__)
56
55
 
57
56
  DOCKER_IMAGE_WORKDIR = "/app"
58
- DOCKER_IMAGE_ZENML_CONFIG_DIR = ".zenconfig"
59
- DOCKER_IMAGE_ZENML_CONFIG_PATH = (
60
- f"{DOCKER_IMAGE_WORKDIR}/{DOCKER_IMAGE_ZENML_CONFIG_DIR}"
61
- )
57
+ DOCKER_IMAGE_ZENML_CONFIG_PATH = f"{DOCKER_IMAGE_WORKDIR}/.zenconfig"
62
58
 
63
59
  DEFAULT_DOCKER_PARENT_IMAGE = (
64
60
  f"zenmldocker/zenml:{zenml.__version__}-"
@@ -334,7 +330,6 @@ class PipelineDockerImageBuilder:
334
330
  dockerfile = self._generate_zenml_pipeline_dockerfile(
335
331
  parent_image=parent_image,
336
332
  docker_settings=docker_settings,
337
- download_files=download_files,
338
333
  requirements_files=requirements_files,
339
334
  apt_packages=apt_packages,
340
335
  entrypoint=entrypoint,
@@ -563,7 +558,6 @@ class PipelineDockerImageBuilder:
563
558
  def _generate_zenml_pipeline_dockerfile(
564
559
  parent_image: str,
565
560
  docker_settings: DockerSettings,
566
- download_files: bool,
567
561
  requirements_files: Sequence[Tuple[str, str, List[str]]] = (),
568
562
  apt_packages: Sequence[str] = (),
569
563
  entrypoint: Optional[str] = None,
@@ -573,7 +567,6 @@ class PipelineDockerImageBuilder:
573
567
  Args:
574
568
  parent_image: The image to use as parent for the Dockerfile.
575
569
  docker_settings: Docker settings for this image build.
576
- download_files: Whether to download files in the build context.
577
570
  requirements_files: List of tuples that contain three items:
578
571
  - the name of a requirements file,
579
572
  - the content of that file,
@@ -641,9 +634,6 @@ class PipelineDockerImageBuilder:
641
634
  )
642
635
 
643
636
  lines.append(f"ENV {ENV_ZENML_ENABLE_REPO_INIT_WARNINGS}=False")
644
- if download_files:
645
- lines.append(f"ENV {ENV_ZENML_REQUIRES_CODE_DOWNLOAD}=True")
646
-
647
637
  lines.append(
648
638
  f"ENV {ENV_ZENML_CONFIG_PATH}={DOCKER_IMAGE_ZENML_CONFIG_PATH}"
649
639
  )
@@ -297,7 +297,7 @@ def model_validator_data_handler(
297
297
  this can create conflicts after the migration and this function will be
298
298
  used as a helper function to handle different types of raw input data.
299
299
 
300
- A code snippet to showcase how the behaviour changes. The "before" validator
300
+ A code snippet to showcase how the behavior changes. The "before" validator
301
301
  prints the type of the input:
302
302
 
303
303
  class Base(BaseModel):
@@ -336,7 +336,7 @@ def model_validator_data_handler(
336
336
  # raw data such as MyClass(...) or MyClass.model_validate()
337
337
 
338
338
  if isinstance(raw_data, dict):
339
- # In most cases, this is the behaviour as the raw input is a dict
339
+ # In most cases, this is the behavior as the raw input is a dict
340
340
  return raw_data
341
341
 
342
342
  elif isinstance(raw_data, base_class):
@@ -354,7 +354,7 @@ def model_validator_data_handler(
354
354
  f"During the validation of a `{base_class}` object, an instance"
355
355
  f"of `{raw_data.__class__}` (super class of `{base_class}`) "
356
356
  f"has been passed as raw input. This might lead to unexpected "
357
- f"behaviour in case `{base_class}` have features which can not"
357
+ f"behavior in case `{base_class}` have features which can not"
358
358
  f"be extracted from an instance of a `{raw_data.__class__}`."
359
359
  )
360
360
  return dict(raw_data)