zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -17,8 +17,8 @@ import os
17
17
  from tempfile import TemporaryDirectory
18
18
  from typing import Any, ClassVar, Tuple, Type
19
19
 
20
- from transformers import AutoTokenizer # type: ignore [import-untyped]
21
- from transformers.tokenization_utils_base import ( # type: ignore [import-untyped]
20
+ from transformers import AutoTokenizer
21
+ from transformers.tokenization_utils_base import (
22
22
  PreTrainedTokenizerBase,
23
23
  )
24
24
 
@@ -17,14 +17,15 @@
17
17
  """Step function to run any ZenML step using Accelerate."""
18
18
 
19
19
  import functools
20
- from typing import Any, Callable, Optional, TypeVar, cast
20
+ from typing import Any, Callable, Dict, Optional, TypeVar, Union, cast
21
21
 
22
22
  import cloudpickle as pickle
23
- from accelerate.commands.launch import ( # type: ignore[import-untyped]
23
+ from accelerate.commands.launch import (
24
24
  launch_command,
25
25
  launch_command_parser,
26
26
  )
27
27
 
28
+ from zenml import get_pipeline_context
28
29
  from zenml.logger import get_logger
29
30
  from zenml.steps import BaseStep
30
31
  from zenml.utils.function_utils import _cli_arg_name, create_cli_wrapped_script
@@ -34,10 +35,9 @@ F = TypeVar("F", bound=Callable[..., Any])
34
35
 
35
36
 
36
37
  def run_with_accelerate(
37
- step_function: BaseStep,
38
- num_processes: Optional[int] = None,
39
- use_cpu: bool = False,
40
- ) -> BaseStep:
38
+ step_function_top_level: Optional[BaseStep] = None,
39
+ **accelerate_launch_kwargs: Any,
40
+ ) -> Union[Callable[[BaseStep], BaseStep], BaseStep]:
41
41
  """Run a function with accelerate.
42
42
 
43
43
  Accelerate package: https://huggingface.co/docs/accelerate/en/index
@@ -45,6 +45,8 @@ def run_with_accelerate(
45
45
  ```python
46
46
  from zenml import step, pipeline
47
47
  from zenml.integrations.hugginface.steps import run_with_accelerate
48
+
49
+ @run_with_accelerate(num_processes=4, multi_gpu=True)
48
50
  @step
49
51
  def training_step(some_param: int, ...):
50
52
  # your training code is below
@@ -52,92 +54,113 @@ def run_with_accelerate(
52
54
 
53
55
  @pipeline
54
56
  def training_pipeline(some_param: int, ...):
55
- run_with_accelerate(training_step, num_processes=4)(some_param, ...)
57
+ training_step(some_param, ...)
56
58
  ```
57
59
 
58
60
  Args:
59
- step_function: The step function to run.
60
- num_processes: The number of processes to use.
61
- use_cpu: Whether to use the CPU.
61
+ step_function_top_level: The step function to run with accelerate [optional].
62
+ Used in functional calls like `run_with_accelerate(some_func,foo=bar)()`.
63
+ accelerate_launch_kwargs: A dictionary of arguments to pass along to the
64
+ `accelerate launch` command, including hardware selection, resource
65
+ allocation, and training paradigm options. Visit
66
+ https://huggingface.co/docs/accelerate/en/package_reference/cli#accelerate-launch
67
+ for more details.
62
68
 
63
69
  Returns:
64
70
  The accelerate-enabled version of the step.
65
71
  """
66
72
 
67
- def _decorator(entrypoint: F) -> F:
68
- @functools.wraps(entrypoint)
69
- def inner(*args: Any, **kwargs: Any) -> Any:
70
- if args:
71
- raise ValueError(
72
- "Accelerated steps do not support positional arguments."
73
- )
74
-
75
- if not use_cpu:
76
- import torch
77
-
78
- logger.info("Starting accelerate job...")
79
-
80
- device_count = torch.cuda.device_count()
81
- if num_processes is None:
82
- _num_processes = device_count
83
- else:
84
- if num_processes > device_count:
85
- logger.warning(
86
- f"Number of processes ({num_processes}) is greater than "
87
- f"the number of available GPUs ({device_count}). Using all GPUs."
88
- )
89
- _num_processes = device_count
90
- else:
91
- _num_processes = num_processes
92
- else:
93
- _num_processes = num_processes or 1
94
-
95
- with create_cli_wrapped_script(
96
- entrypoint, flavour="accelerate"
97
- ) as (
98
- script_path,
99
- output_path,
100
- ):
101
- commands = ["--num_processes", str(_num_processes)]
102
- if use_cpu:
103
- commands += [
104
- "--cpu",
105
- "--num_cpu_threads_per_process",
106
- "10",
107
- ]
108
- commands.append(str(script_path.absolute()))
109
- for k, v in kwargs.items():
110
- k = _cli_arg_name(k)
111
- if isinstance(v, bool):
112
- if v:
113
- commands.append(f"--{k}")
114
- elif type(v) in (list, tuple, set):
115
- for each in v:
116
- commands += [f"--{k}", f"{each}"]
117
- else:
118
- commands += [f"--{k}", f"{v}"]
119
-
120
- logger.debug(commands)
121
-
122
- parser = launch_command_parser()
123
- args = parser.parse_args(commands)
124
- try:
125
- launch_command(args)
126
- except Exception as e:
127
- logger.error(
128
- "Accelerate training job failed... See error message for details."
73
+ def _decorator(step_function: BaseStep) -> BaseStep:
74
+ def _wrapper(
75
+ entrypoint: F, accelerate_launch_kwargs: Dict[str, Any]
76
+ ) -> F:
77
+ @functools.wraps(entrypoint)
78
+ def inner(*args: Any, **kwargs: Any) -> Any:
79
+ if args:
80
+ raise ValueError(
81
+ "Accelerated steps do not support positional arguments."
129
82
  )
130
- raise RuntimeError(
131
- "Accelerate training job failed."
132
- ) from e
133
- else:
134
- logger.info(
135
- "Accelerate training job finished successfully."
136
- )
137
- return pickle.load(open(output_path, "rb"))
138
-
139
- return cast(F, inner)
140
-
141
- setattr(step_function, "entrypoint", _decorator(step_function.entrypoint))
142
83
 
143
- return step_function
84
+ with create_cli_wrapped_script(
85
+ entrypoint, flavor="accelerate"
86
+ ) as (
87
+ script_path,
88
+ output_path,
89
+ ):
90
+ commands = [str(script_path.absolute())]
91
+ for k, v in kwargs.items():
92
+ k = _cli_arg_name(k)
93
+ if isinstance(v, bool):
94
+ if v:
95
+ commands.append(f"--{k}")
96
+ elif type(v) in (list, tuple, set):
97
+ for each in v:
98
+ commands += [f"--{k}", f"{each}"]
99
+ else:
100
+ commands += [f"--{k}", f"{v}"]
101
+ logger.debug(commands)
102
+
103
+ parser = launch_command_parser()
104
+ args = parser.parse_args(commands)
105
+ for k, v in accelerate_launch_kwargs.items():
106
+ if k in args:
107
+ setattr(args, k, v)
108
+ else:
109
+ logger.warning(
110
+ f"You passed in `{k}` as an `accelerate launch` argument, but it was not accepted. "
111
+ "Please check https://huggingface.co/docs/accelerate/en/package_reference/cli#accelerate-launch "
112
+ "to find out more about supported arguments and retry."
113
+ )
114
+ try:
115
+ launch_command(args)
116
+ except Exception as e:
117
+ logger.error(
118
+ "Accelerate training job failed... See error message for details."
119
+ )
120
+ raise RuntimeError(
121
+ "Accelerate training job failed."
122
+ ) from e
123
+ else:
124
+ logger.info(
125
+ "Accelerate training job finished successfully."
126
+ )
127
+ return pickle.load(open(output_path, "rb"))
128
+
129
+ return cast(F, inner)
130
+
131
+ try:
132
+ get_pipeline_context()
133
+ except RuntimeError:
134
+ pass
135
+ else:
136
+ raise RuntimeError(
137
+ f"`{run_with_accelerate.__name__}` decorator cannot be used "
138
+ "in a functional way with steps, please apply decoration "
139
+ "directly to a step instead. This behavior will be also "
140
+ "allowed in future, but now it faces technical limitations.\n"
141
+ "Example (allowed):\n"
142
+ f"@{run_with_accelerate.__name__}(...)\n"
143
+ f"def {step_function.name}(...):\n"
144
+ " ...\n"
145
+ "Example (not allowed):\n"
146
+ "def my_pipeline(...):\n"
147
+ f" run_with_accelerate({step_function.name},...)(...)\n"
148
+ )
149
+
150
+ setattr(
151
+ step_function, "unwrapped_entrypoint", step_function.entrypoint
152
+ )
153
+ setattr(
154
+ step_function,
155
+ "entrypoint",
156
+ _wrapper(
157
+ step_function.entrypoint,
158
+ accelerate_launch_kwargs=accelerate_launch_kwargs,
159
+ ),
160
+ )
161
+
162
+ return step_function
163
+
164
+ if step_function_top_level:
165
+ return _decorator(step_function_top_level)
166
+ return _decorator
@@ -81,6 +81,15 @@ class HyperAIOrchestratorConfig(
81
81
  """
82
82
  return True
83
83
 
84
+ @property
85
+ def is_schedulable(self) -> bool:
86
+ """Whether the orchestrator is schedulable or not.
87
+
88
+ Returns:
89
+ Whether the orchestrator is schedulable or not.
90
+ """
91
+ return True
92
+
84
93
 
85
94
  class HyperAIOrchestratorFlavor(BaseOrchestratorFlavor):
86
95
  """Flavor for the HyperAI orchestrator."""
@@ -213,6 +213,15 @@ class KubeflowOrchestratorConfig(
213
213
  """
214
214
  return self.synchronous
215
215
 
216
+ @property
217
+ def is_schedulable(self) -> bool:
218
+ """Whether the orchestrator is schedulable or not.
219
+
220
+ Returns:
221
+ Whether the orchestrator is schedulable or not.
222
+ """
223
+ return True
224
+
216
225
 
217
226
  class KubeflowOrchestratorFlavor(BaseOrchestratorFlavor):
218
227
  """Kubeflow orchestrator flavor."""
@@ -32,7 +32,16 @@
32
32
 
33
33
  import os
34
34
  import types
35
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
35
+ from typing import (
36
+ TYPE_CHECKING,
37
+ Any,
38
+ Dict,
39
+ List,
40
+ Optional,
41
+ Tuple,
42
+ Type,
43
+ cast,
44
+ )
36
45
  from uuid import UUID
37
46
 
38
47
  import kfp
@@ -125,6 +125,15 @@ class KubernetesOrchestratorConfig(
125
125
  """
126
126
  return self.synchronous
127
127
 
128
+ @property
129
+ def is_schedulable(self) -> bool:
130
+ """Whether the orchestrator is schedulable or not.
131
+
132
+ Returns:
133
+ Whether the orchestrator is schedulable or not.
134
+ """
135
+ return True
136
+
128
137
 
129
138
  class KubernetesOrchestratorFlavor(BaseOrchestratorFlavor):
130
139
  """Kubernetes orchestrator flavor."""
@@ -31,7 +31,16 @@
31
31
  """Kubernetes-native orchestrator."""
32
32
 
33
33
  import os
34
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
34
+ from typing import (
35
+ TYPE_CHECKING,
36
+ Any,
37
+ Dict,
38
+ List,
39
+ Optional,
40
+ Tuple,
41
+ Type,
42
+ cast,
43
+ )
35
44
 
36
45
  from kubernetes import client as k8s_client
37
46
  from kubernetes import config as k8s_config
@@ -0,0 +1,48 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Initialization of the Lightning integration for ZenML."""
15
+
16
+ from typing import List, Type
17
+
18
+ from zenml.integrations.constants import (
19
+ LIGHTNING,
20
+ )
21
+ from zenml.integrations.integration import Integration
22
+ from zenml.stack import Flavor
23
+
24
+ LIGHTNING_ORCHESTRATOR_FLAVOR = "lightning"
25
+
26
+
27
+ class LightningIntegration(Integration):
28
+ """Definition of Lightning Integration for ZenML."""
29
+
30
+ NAME = LIGHTNING
31
+ REQUIREMENTS = ["lightning-sdk"]
32
+
33
+ @classmethod
34
+ def flavors(cls) -> List[Type[Flavor]]:
35
+ """Declare the stack component flavors for the Lightning integration.
36
+
37
+ Returns:
38
+ List of stack component flavors for this integration.
39
+ """
40
+ from zenml.integrations.lightning.flavors import (
41
+ LightningOrchestratorFlavor,
42
+ )
43
+
44
+ return [
45
+ LightningOrchestratorFlavor,
46
+ ]
47
+
48
+ LightningIntegration.check_installation()
@@ -0,0 +1,23 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Lightning integration flavors."""
15
+
16
+ from zenml.integrations.lightning.flavors.lightning_orchestrator_flavor import (
17
+ LightningOrchestratorConfig,
18
+ LightningOrchestratorFlavor,
19
+ )
20
+ __all__ = [
21
+ "LightningOrchestratorFlavor",
22
+ "LightningOrchestratorConfig",
23
+ ]
@@ -0,0 +1,148 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Lightning orchestrator base config and settings."""
15
+
16
+ from typing import TYPE_CHECKING, List, Optional, Type
17
+
18
+ from zenml.config.base_settings import BaseSettings
19
+ from zenml.integrations.lightning import LIGHTNING_ORCHESTRATOR_FLAVOR
20
+ from zenml.logger import get_logger
21
+ from zenml.orchestrators import BaseOrchestratorConfig
22
+ from zenml.orchestrators.base_orchestrator import BaseOrchestratorFlavor
23
+ from zenml.utils.secret_utils import SecretField
24
+
25
+ if TYPE_CHECKING:
26
+ from zenml.integrations.lightning.orchestrators import (
27
+ LightningOrchestrator,
28
+ )
29
+
30
+ logger = get_logger(__name__)
31
+
32
+
33
+ class LightningOrchestratorSettings(BaseSettings):
34
+ """Lightning orchestrator base settings.
35
+
36
+ Attributes:
37
+ main_studio_name: Main studio name.
38
+ machine_type: Machine type.
39
+ user_id: User id.
40
+ api_key: api_key.
41
+ username: Username.
42
+ teamspace: Teamspace.
43
+ organization: Organization.
44
+ custom_commands: Custom commands to run.
45
+ synchronous: If `True`, the client running a pipeline using this
46
+ orchestrator waits until all steps finish running. If `False`,
47
+ the client returns immediately and the pipeline is executed
48
+ asynchronously. Defaults to `True`. This setting only
49
+ has an effect when specified on the pipeline and will be ignored if
50
+ specified on steps.
51
+ """
52
+
53
+ # Resources
54
+ main_studio_name: Optional[str] = None
55
+ machine_type: Optional[str] = None
56
+ user_id: Optional[str] = SecretField(default=None)
57
+ api_key: Optional[str] = SecretField(default=None)
58
+ username: Optional[str] = None
59
+ teamspace: Optional[str] = None
60
+ organization: Optional[str] = None
61
+ custom_commands: Optional[List[str]] = None
62
+ synchronous: bool = True
63
+
64
+
65
+ class LightningOrchestratorConfig(
66
+ BaseOrchestratorConfig, LightningOrchestratorSettings
67
+ ):
68
+ """Lightning orchestrator base config."""
69
+
70
+ @property
71
+ def is_local(self) -> bool:
72
+ """Checks if this stack component is running locally.
73
+
74
+ Returns:
75
+ True if this config is for a local component, False otherwise.
76
+ """
77
+ return False
78
+
79
+ @property
80
+ def is_synchronous(self) -> bool:
81
+ """Whether the orchestrator runs synchronous or not.
82
+
83
+ Returns:
84
+ Whether the orchestrator runs synchronous or not.
85
+ """
86
+ return self.synchronous
87
+
88
+
89
+ class LightningOrchestratorFlavor(BaseOrchestratorFlavor):
90
+ """Lightning orchestrator flavor."""
91
+
92
+ @property
93
+ def name(self) -> str:
94
+ """Name of the flavor.
95
+
96
+ Returns:
97
+ The name of the flavor.
98
+ """
99
+ return LIGHTNING_ORCHESTRATOR_FLAVOR
100
+
101
+ @property
102
+ def docs_url(self) -> Optional[str]:
103
+ """A url to point at docs explaining this flavor.
104
+
105
+ Returns:
106
+ A flavor docs url.
107
+ """
108
+ return self.generate_default_docs_url()
109
+
110
+ @property
111
+ def sdk_docs_url(self) -> Optional[str]:
112
+ """A url to point at SDK docs explaining this flavor.
113
+
114
+ Returns:
115
+ A flavor SDK docs url.
116
+ """
117
+ return self.generate_default_sdk_docs_url()
118
+
119
+ @property
120
+ def logo_url(self) -> str:
121
+ """A url to represent the flavor in the dashboard.
122
+
123
+ Returns:
124
+ The flavor logo.
125
+ """
126
+ return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/orchestrator/lightning.png"
127
+
128
+ @property
129
+ def config_class(self) -> Type[LightningOrchestratorConfig]:
130
+ """Returns `KubeflowOrchestratorConfig` config class.
131
+
132
+ Returns:
133
+ The config class.
134
+ """
135
+ return LightningOrchestratorConfig
136
+
137
+ @property
138
+ def implementation_class(self) -> Type["LightningOrchestrator"]:
139
+ """Implementation class for this flavor.
140
+
141
+ Returns:
142
+ The implementation class.
143
+ """
144
+ from zenml.integrations.lightning.orchestrators import (
145
+ LightningOrchestrator,
146
+ )
147
+
148
+ return LightningOrchestrator
@@ -0,0 +1,23 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Initialization of the Lightning ZenML orchestrator."""
15
+
16
+ from zenml.integrations.lightning.orchestrators.lightning_orchestrator import ( # noqa
17
+ LightningOrchestrator,
18
+ )
19
+ from zenml.integrations.lightning.orchestrators.lightning_orchestrator_entrypoint_configuration import (
20
+ LightningOrchestratorEntrypointConfiguration,
21
+ )
22
+
23
+ __all__ = ["LightningOrchestrator", "LightningOrchestratorEntrypointConfiguration"]