zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,139 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """AzureML definitions."""
15
+
16
+ from typing import Optional
17
+
18
+ from pydantic import model_validator
19
+
20
+ from zenml.config.base_settings import BaseSettings
21
+ from zenml.logger import get_logger
22
+ from zenml.utils.enum_utils import StrEnum
23
+
24
+ logger = get_logger(__name__)
25
+
26
+
27
+ class AzureMLComputeTypes(StrEnum):
28
+ """Enum for different types of compute on AzureML."""
29
+
30
+ SERVERLESS = "serverless"
31
+ COMPUTE_INSTANCE = "compute-instance"
32
+ COMPUTE_CLUSTER = "compute-cluster"
33
+
34
+
35
+ class AzureMLComputeSettings(BaseSettings):
36
+ """Settings for the AzureML compute.
37
+
38
+ These settings adjust the compute resources that will be used by the
39
+ pipeline execution.
40
+
41
+ There are three possible use cases for this implementation:
42
+
43
+ 1. Serverless compute (default behavior):
44
+ - The `mode` is set to `serverless` (default behavior).
45
+ - All the other parameters become irrelevant and will throw a
46
+ warning if set.
47
+
48
+ 2. Compute instance:
49
+ - The `mode` is set to `compute-instance`.
50
+ - In this case, users have to provide a `compute-name`.
51
+ - If a compute instance exists with this name, this instance
52
+ will be used and all the other parameters become irrelevant
53
+ and will throw a warning if set.
54
+ - If a compute instance does not already exist, ZenML will
55
+ create it. You can use the parameters `compute_size` and
56
+ `idle_type_before_shutdown_minutes` for this operation.
57
+
58
+ 3. Compute cluster:
59
+ - The `mode` is set to `compute-cluster`.
60
+ - In this case, users have to provide a `compute-name`.
61
+ - If a compute cluster exists with this name, this instance
62
+ will be used and all the other parameters become irrelevant
63
+ and will throw a warning if set.
64
+ - If a compute cluster does not already exist, ZenML will
65
+ create it. You can set the additional parameters for this
66
+ operation.
67
+ """
68
+
69
+ # Mode for compute
70
+ mode: AzureMLComputeTypes = AzureMLComputeTypes.SERVERLESS
71
+
72
+ # Common Configuration for Compute Instances and Clusters
73
+ compute_name: Optional[str] = None
74
+ size: Optional[str] = None
75
+
76
+ # Additional configuration for a Compute Instance
77
+ idle_time_before_shutdown_minutes: Optional[int] = None
78
+
79
+ # Additional configuration for a Compute Cluster
80
+ idle_time_before_scaledown_down: Optional[int] = None
81
+ location: Optional[str] = None
82
+ min_instances: Optional[int] = None
83
+ max_instances: Optional[int] = None
84
+ tier: Optional[str] = None
85
+
86
+ @model_validator(mode="after")
87
+ def azureml_settings_validator(self) -> "AzureMLComputeSettings":
88
+ """Checks whether the right configuration is set based on mode.
89
+
90
+ Returns:
91
+ the instance itself.
92
+
93
+ Raises:
94
+ AssertionError: if a name is not provided when working with
95
+ instances and clusters.
96
+ """
97
+ viable_configuration_fields = {
98
+ AzureMLComputeTypes.SERVERLESS: {"mode"},
99
+ AzureMLComputeTypes.COMPUTE_INSTANCE: {
100
+ "mode",
101
+ "compute_name",
102
+ "size",
103
+ "idle_time_before_shutdown_minutes",
104
+ },
105
+ AzureMLComputeTypes.COMPUTE_CLUSTER: {
106
+ "mode",
107
+ "compute_name",
108
+ "size",
109
+ "idle_time_before_scaledown_down",
110
+ "location",
111
+ "min_instances",
112
+ "max_instances",
113
+ "tier",
114
+ },
115
+ }
116
+ viable_fields = viable_configuration_fields[self.mode]
117
+
118
+ for field in self.model_fields_set:
119
+ if (
120
+ field not in viable_fields
121
+ and field in AzureMLComputeSettings.model_fields
122
+ ):
123
+ logger.warning(
124
+ f"In the {self.__class__.__name__} settings, the mode of "
125
+ f"operation is set to {self.mode}. In this mode, you can "
126
+ f"not configure the parameter '{field}'. This "
127
+ "configuration will be ignored."
128
+ )
129
+
130
+ if (
131
+ self.mode == AzureMLComputeTypes.COMPUTE_INSTANCE
132
+ or self.mode == AzureMLComputeTypes.COMPUTE_CLUSTER
133
+ ):
134
+ assert self.compute_name is not None, (
135
+ "When you are working with compute instances and clusters, "
136
+ "please define a name for the compute target."
137
+ )
138
+
139
+ return self
@@ -15,20 +15,19 @@
15
15
 
16
16
  from typing import TYPE_CHECKING, Optional, Type
17
17
 
18
- from pydantic import Field, model_validator
18
+ from pydantic import Field
19
19
 
20
- from zenml.config.base_settings import BaseSettings
21
20
  from zenml.integrations.azure import (
22
21
  AZURE_RESOURCE_TYPE,
23
22
  AZUREML_ORCHESTRATOR_FLAVOR,
24
23
  )
24
+ from zenml.integrations.azure.flavors.azureml import AzureMLComputeSettings
25
25
  from zenml.logger import get_logger
26
26
  from zenml.models import ServiceConnectorRequirements
27
27
  from zenml.orchestrators.base_orchestrator import (
28
28
  BaseOrchestratorConfig,
29
29
  BaseOrchestratorFlavor,
30
30
  )
31
- from zenml.utils.enum_utils import StrEnum
32
31
 
33
32
  if TYPE_CHECKING:
34
33
  from zenml.integrations.azure.orchestrators import AzureMLOrchestrator
@@ -36,118 +35,8 @@ if TYPE_CHECKING:
36
35
  logger = get_logger(__name__)
37
36
 
38
37
 
39
- class AzureMLComputeTypes(StrEnum):
40
- """Enum for different types of compute on AzureML."""
41
-
42
- SERVERLESS = "serverless"
43
- COMPUTE_INSTANCE = "compute-instance"
44
- COMPUTE_CLUSTER = "compute-cluster"
45
-
46
-
47
- class AzureMLOrchestratorSettings(BaseSettings):
48
- """Settings for the AzureML orchestrator.
49
-
50
- These settings adjust the compute resources that will be used by the
51
- pipeline execution.
52
-
53
- There are three possible use cases for this implementation:
54
-
55
- 1. Serverless compute (default behaviour):
56
- - The `mode` is set to `serverless` (default behaviour).
57
- - All the other parameters become irrelevant and will throw a
58
- warning if set.
59
-
60
- 2. Compute instance:
61
- - The `mode` is set to `compute-instance`.
62
- - In this case, users have to provide a `compute-name`.
63
- - If a compute instance exists with this name, this instance
64
- will be used and all the other parameters become irrelevant
65
- and will throw a warning if set.
66
- - If a compute instance does not already exist, ZenML will
67
- create it. You can use the parameters `compute_size` and
68
- `idle_type_before_shutdown_minutes` for this operation.
69
-
70
- 3. Compute cluster:
71
- - The `mode` is set to `compute-cluster`.
72
- - In this case, users have to provide a `compute-name`.
73
- - If a compute cluster exists with this name, this instance
74
- will be used and all the other parameters become irrelevant
75
- and will throw a warning if set.
76
- - If a compute cluster does not already exist, ZenML will
77
- create it. You can all the additional parameters for this
78
- operation.
79
- """
80
-
81
- # Mode for compute
82
- mode: AzureMLComputeTypes = AzureMLComputeTypes.SERVERLESS
83
-
84
- # Common Configuration for Compute Instances and Clusters
85
- compute_name: Optional[str] = None
86
- size: Optional[str] = None
87
-
88
- # Additional configuration for a Compute Instance
89
- idle_time_before_shutdown_minutes: Optional[int] = None
90
-
91
- # Additional configuration for a Compute Cluster
92
- idle_time_before_scaledown_down: Optional[int] = None
93
- location: Optional[str] = None
94
- min_instances: Optional[int] = None
95
- max_instances: Optional[int] = None
96
- tier: Optional[str] = None
97
-
98
- @model_validator(mode="after")
99
- def azureml_settings_validator(self) -> "AzureMLOrchestratorSettings":
100
- """Checks whether the right configuration is set based on mode.
101
-
102
- Returns:
103
- the instance itself.
104
-
105
- Raises:
106
- AssertionError: if a name is not provided when working with
107
- instances and clusters.
108
- """
109
- excluded_fields = {"subscription_id", "resource_group", "workspace"}
110
-
111
- viable_configuration_fields = {
112
- AzureMLComputeTypes.SERVERLESS: {"mode"},
113
- AzureMLComputeTypes.COMPUTE_INSTANCE: {
114
- "mode",
115
- "compute_name",
116
- "size",
117
- "idle_time_before_shutdown_minutes",
118
- },
119
- AzureMLComputeTypes.COMPUTE_CLUSTER: {
120
- "mode",
121
- "compute_name",
122
- "size",
123
- "idle_time_before_scaledown_down",
124
- "location",
125
- "min_instances",
126
- "max_instances",
127
- "tier",
128
- },
129
- }
130
- viable_fields = viable_configuration_fields[self.mode]
131
-
132
- for field in self.model_fields_set:
133
- if field not in viable_fields and field not in excluded_fields:
134
- logger.warning(
135
- "In the AzureML Orchestrator Settings, the mode of "
136
- f"operation is set to {self.mode}. In this mode, you can "
137
- f"not configure the parameter '{field}'. This "
138
- "configuration will be ignored."
139
- )
140
-
141
- if (
142
- self.mode == AzureMLComputeTypes.COMPUTE_INSTANCE
143
- or self.mode == AzureMLComputeTypes.COMPUTE_CLUSTER
144
- ):
145
- assert self.compute_name is not None, (
146
- "When you are working with compute instances and clusters, "
147
- "please define a name for the compute target."
148
- )
149
-
150
- return self
38
+ class AzureMLOrchestratorSettings(AzureMLComputeSettings):
39
+ """Settings for the AzureML orchestrator."""
151
40
 
152
41
 
153
42
  class AzureMLOrchestratorConfig(
@@ -164,6 +53,10 @@ class AzureMLOrchestratorConfig(
164
53
  workspace: str = Field(
165
54
  description="Name of the workspace that AzureML is running on."
166
55
  )
56
+ synchronous: bool = Field(
57
+ default=True,
58
+ description="Whether the orchestrator runs synchronously or not.",
59
+ )
167
60
 
168
61
  @property
169
62
  def is_remote(self) -> bool:
@@ -180,12 +73,21 @@ class AzureMLOrchestratorConfig(
180
73
 
181
74
  @property
182
75
  def is_synchronous(self) -> bool:
183
- """Whether the orchestrator runs synchronous or not.
76
+ """Whether the orchestrator runs synchronously or not.
184
77
 
185
78
  Returns:
186
- Whether the orchestrator runs synchronous or not.
79
+ Whether the orchestrator runs synchronously or not.
187
80
  """
188
- return False
81
+ return self.synchronous
82
+
83
+ @property
84
+ def is_schedulable(self) -> bool:
85
+ """Whether the orchestrator is schedulable or not.
86
+
87
+ Returns:
88
+ Whether the orchestrator is schedulable or not.
89
+ """
90
+ return True
189
91
 
190
92
 
191
93
  class AzureMLOrchestratorFlavor(BaseOrchestratorFlavor):
@@ -13,29 +13,65 @@
13
13
  # permissions and limitations under the License.
14
14
  """AzureML step operator flavor."""
15
15
 
16
- from typing import TYPE_CHECKING, Optional, Type
16
+ from typing import TYPE_CHECKING, Any, Dict, Optional, Type
17
17
 
18
- from zenml.config.base_settings import BaseSettings
19
- from zenml.integrations.azure import AZUREML_STEP_OPERATOR_FLAVOR
18
+ from pydantic import Field, model_validator
19
+
20
+ from zenml.integrations.azure import (
21
+ AZURE_RESOURCE_TYPE,
22
+ AZUREML_STEP_OPERATOR_FLAVOR,
23
+ )
24
+ from zenml.integrations.azure.flavors.azureml import (
25
+ AzureMLComputeSettings,
26
+ AzureMLComputeTypes,
27
+ )
28
+ from zenml.models import ServiceConnectorRequirements
20
29
  from zenml.step_operators.base_step_operator import (
21
30
  BaseStepOperatorConfig,
22
31
  BaseStepOperatorFlavor,
23
32
  )
33
+ from zenml.utils.pydantic_utils import before_validator_handler
24
34
  from zenml.utils.secret_utils import SecretField
25
35
 
26
36
  if TYPE_CHECKING:
27
37
  from zenml.integrations.azure.step_operators import AzureMLStepOperator
28
38
 
29
39
 
30
- class AzureMLStepOperatorSettings(BaseSettings):
40
+ class AzureMLStepOperatorSettings(AzureMLComputeSettings):
31
41
  """Settings for the AzureML step operator.
32
42
 
33
43
  Attributes:
34
- environment_name: The name of the environment if there
35
- already exists one.
44
+ compute_target_name: The name of the configured ComputeTarget.
45
+ Deprecated in favor of `compute_name`.
36
46
  """
37
47
 
38
- environment_name: Optional[str] = None
48
+ compute_target_name: Optional[str] = Field(
49
+ default=None,
50
+ description="Name of the configured ComputeTarget. Deprecated in favor "
51
+ "of `compute_name`.",
52
+ )
53
+
54
+ @model_validator(mode="before")
55
+ @classmethod
56
+ @before_validator_handler
57
+ def _migrate_compute_name(cls, data: Dict[str, Any]) -> Dict[str, Any]:
58
+ """Backward compatibility for compute_target_name.
59
+
60
+ Args:
61
+ data: The model data.
62
+
63
+ Returns:
64
+ The migrated data.
65
+ """
66
+ if (
67
+ "compute_target_name" in data
68
+ and "compute_name" not in data
69
+ and "mode" not in data
70
+ ):
71
+ data["compute_name"] = data.pop("compute_target_name")
72
+ data["mode"] = AzureMLComputeTypes.COMPUTE_INSTANCE
73
+
74
+ return data
39
75
 
40
76
 
41
77
  class AzureMLStepOperatorConfig(
@@ -48,19 +84,21 @@ class AzureMLStepOperatorConfig(
48
84
  resource_group: The resource group to which the AzureML workspace
49
85
  is deployed.
50
86
  workspace_name: The name of the AzureML Workspace.
51
- compute_target_name: The name of the configured ComputeTarget.
52
- An instance of it has to be created on the portal if it doesn't
53
- exist already.
54
87
  tenant_id: The Azure Tenant ID.
55
88
  service_principal_id: The ID for the service principal that is created
56
89
  to allow apps to access secure resources.
57
90
  service_principal_password: Password for the service principal.
58
91
  """
59
92
 
60
- subscription_id: str
61
- resource_group: str
62
- workspace_name: str
63
- compute_target_name: str
93
+ subscription_id: str = Field(
94
+ description="Subscription ID that AzureML is running on."
95
+ )
96
+ resource_group: str = Field(
97
+ description="Name of the resource group that AzureML is running on.",
98
+ )
99
+ workspace_name: str = Field(
100
+ description="Name of the workspace that AzureML is running on."
101
+ )
64
102
 
65
103
  # Service principal authentication
66
104
  # https://docs.microsoft.com/en-us/azure/machine-learning/how-to-setup-authentication#configure-a-service-principal
@@ -94,6 +132,21 @@ class AzureMLStepOperatorFlavor(BaseStepOperatorFlavor):
94
132
  """
95
133
  return AZUREML_STEP_OPERATOR_FLAVOR
96
134
 
135
+ @property
136
+ def service_connector_requirements(
137
+ self,
138
+ ) -> Optional[ServiceConnectorRequirements]:
139
+ """Service connector resource requirements for service connectors.
140
+
141
+ Specifies resource requirements that are used to filter the available
142
+ service connector types that are compatible with this flavor.
143
+
144
+ Returns:
145
+ Requirements for compatible service connectors, if a service
146
+ connector is required for this flavor.
147
+ """
148
+ return ServiceConnectorRequirements(resource_type=AZURE_RESOURCE_TYPE)
149
+
97
150
  @property
98
151
  def docs_url(self) -> Optional[str]:
99
152
  """A url to point at docs explaining this flavor.