zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -26,6 +26,7 @@ from typing import (
26
26
  Union,
27
27
  cast,
28
28
  )
29
+ from uuid import UUID
29
30
 
30
31
  from azure.ai.ml import Input, MLClient, Output
31
32
  from azure.ai.ml.constants import TimeZone
@@ -40,15 +41,16 @@ from azure.ai.ml.entities import (
40
41
  from azure.core.exceptions import (
41
42
  HttpResponseError,
42
43
  ResourceExistsError,
43
- ResourceNotFoundError,
44
44
  )
45
45
  from azure.identity import DefaultAzureCredential
46
46
 
47
47
  from zenml.config.base_settings import BaseSettings
48
48
  from zenml.config.step_configurations import Step
49
+ from zenml.constants import METADATA_ORCHESTRATOR_URL
49
50
  from zenml.enums import StackComponentType
51
+ from zenml.integrations.azure.azureml_utils import create_or_get_compute
52
+ from zenml.integrations.azure.flavors.azureml import AzureMLComputeTypes
50
53
  from zenml.integrations.azure.flavors.azureml_orchestrator_flavor import (
51
- AzureMLComputeTypes,
52
54
  AzureMLOrchestratorConfig,
53
55
  AzureMLOrchestratorSettings,
54
56
  )
@@ -56,14 +58,13 @@ from zenml.integrations.azure.orchestrators.azureml_orchestrator_entrypoint_conf
56
58
  AzureMLEntrypointConfiguration,
57
59
  )
58
60
  from zenml.logger import get_logger
61
+ from zenml.metadata.metadata_types import MetadataType, Uri
59
62
  from zenml.orchestrators import ContainerizedOrchestrator
60
63
  from zenml.orchestrators.utils import get_orchestrator_run_name
61
64
  from zenml.stack import StackValidator
62
65
  from zenml.utils.string_utils import b64_encode
63
66
 
64
67
  if TYPE_CHECKING:
65
- from azure.ai.ml.entities import AmlCompute, ComputeInstance
66
-
67
68
  from zenml.models import PipelineDeploymentResponse
68
69
  from zenml.stack import Stack
69
70
 
@@ -193,172 +194,6 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
193
194
  command=" ".join(command + arguments),
194
195
  )
195
196
 
196
- @staticmethod
197
- def _check_settings_and_compute_configuration(
198
- parameter: str,
199
- settings: AzureMLOrchestratorSettings,
200
- compute: Union["ComputeInstance", "AmlCompute"],
201
- ) -> None:
202
- """Utility function comparing a parameter between settings and compute.
203
-
204
- Args:
205
- parameter: the name of the parameter.
206
- settings: The AzureML orchestrator settings.
207
- compute: The compute instance or cluster from AzureML.
208
- """
209
- # Check the compute size
210
- compute_value = getattr(compute, parameter)
211
- settings_value = getattr(settings, parameter)
212
-
213
- if settings_value is not None and settings_value != compute_value:
214
- logger.warning(
215
- f"The '{parameter}' defined in the settings '{settings_value}' "
216
- "does not match the actual parameter of the instance: "
217
- f"'{compute_value}'. Will ignore this setting for now."
218
- )
219
-
220
- def _create_or_get_compute(
221
- self, client: MLClient, settings: AzureMLOrchestratorSettings
222
- ) -> Optional[str]:
223
- """Creates or fetches the compute target if defined in the settings.
224
-
225
- Args:
226
- client: the AzureML client.
227
- settings: the settings for the orchestrator.
228
-
229
- Returns:
230
- None, if the orchestrator is using serverless compute or
231
- str, the name of the compute target (instance or cluster).
232
-
233
- Raises:
234
- RuntimeError: if the fetched compute target is unsupported or the
235
- mode defined in the setting does not match the type of the
236
- compute target.
237
- """
238
- # If the mode is serverless, we can not fetch anything anyhow
239
- if settings.mode == AzureMLComputeTypes.SERVERLESS:
240
- return None
241
-
242
- # If a name is not provided, generate one based on the orchestrator id
243
- compute_name = settings.compute_name or f"compute_{self.id}"
244
- # Try to fetch the compute target
245
- try:
246
- compute = client.compute.get(compute_name)
247
-
248
- logger.info(f"Using existing compute target: '{compute_name}'.")
249
-
250
- # Check if compute size matches with the settings
251
- self._check_settings_and_compute_configuration(
252
- parameter="size", settings=settings, compute=compute
253
- )
254
-
255
- compute_type = compute.type
256
-
257
- # Check the type and matches the settings
258
- if compute_type == "computeinstance": # Compute Instance
259
- if settings.mode != AzureMLComputeTypes.COMPUTE_INSTANCE:
260
- raise RuntimeError(
261
- "The mode of operation for the compute target defined"
262
- f"in the settings '{settings.mode}' does not match "
263
- f"the type of the compute target: `{compute_name}` "
264
- "which is a 'compute-instance'. Please make sure that "
265
- "the settings are adjusted properly."
266
- )
267
-
268
- if compute.state != "Running":
269
- raise RuntimeError(
270
- f"The compute instance `{compute_name}` is not in a "
271
- "running state at the moment. Please make sure that "
272
- "the compute target is running, before executing the "
273
- "pipeline."
274
- )
275
-
276
- # Idle time before shutdown
277
- self._check_settings_and_compute_configuration(
278
- parameter="idle_time_before_shutdown_minutes",
279
- settings=settings,
280
- compute=compute,
281
- )
282
-
283
- elif compute_type == "amIcompute": # Compute Cluster
284
- if settings.mode != AzureMLComputeTypes.COMPUTE_CLUSTER:
285
- raise RuntimeError(
286
- "The mode of operation for the compute target defined "
287
- f"in the settings '{settings.mode}' does not match "
288
- f"the type of the compute target: `{compute_name}` "
289
- "which is a 'compute-cluster'. Please make sure that "
290
- "the settings are adjusted properly."
291
- )
292
-
293
- if compute.provisioning_state != "Succeeded":
294
- raise RuntimeError(
295
- f"The provisioning state '{compute.provisioning_state}'"
296
- f"of the compute cluster `{compute_name}` is not "
297
- "successful. Please make sure that the compute cluster "
298
- "is provisioned properly, before executing the "
299
- "pipeline."
300
- )
301
-
302
- for parameter in [
303
- "idle_time_before_scale_down",
304
- "max_instances",
305
- "min_instances",
306
- "tier",
307
- "location",
308
- ]:
309
- # Check all possible configurations
310
- self._check_settings_and_compute_configuration(
311
- parameter=parameter, settings=settings, compute=compute
312
- )
313
- else:
314
- raise RuntimeError(f"Unsupported compute type: {compute_type}")
315
- return compute_name
316
-
317
- # If the compute target does not exist create it
318
- except ResourceNotFoundError:
319
- logger.info(
320
- "Can not find the compute target with name: "
321
- f"'{compute_name}':"
322
- )
323
-
324
- if settings.mode == AzureMLComputeTypes.COMPUTE_INSTANCE:
325
- logger.info(
326
- "Creating a new compute instance. This might take a "
327
- "few minutes."
328
- )
329
-
330
- from azure.ai.ml.entities import ComputeInstance
331
-
332
- compute_instance = ComputeInstance(
333
- name=compute_name,
334
- size=settings.size,
335
- idle_time_before_shutdown_minutes=settings.idle_time_before_shutdown_minutes,
336
- )
337
- client.begin_create_or_update(compute_instance).result()
338
- return compute_name
339
-
340
- elif settings.mode == AzureMLComputeTypes.COMPUTE_CLUSTER:
341
- logger.info(
342
- "Creating a new compute cluster. This might take a "
343
- "few minutes."
344
- )
345
-
346
- from azure.ai.ml.entities import AmlCompute
347
-
348
- compute_cluster = AmlCompute(
349
- name=compute_name,
350
- size=settings.size,
351
- location=settings.location,
352
- min_instances=settings.min_instances,
353
- max_instances=settings.max_instances,
354
- idle_time_before_scale_down=settings.idle_time_before_scaledown_down,
355
- tier=settings.tier,
356
- )
357
- client.begin_create_or_update(compute_cluster).result()
358
- return compute_name
359
-
360
- return None
361
-
362
197
  def prepare_or_run_pipeline(
363
198
  self,
364
199
  deployment: "PipelineDeploymentResponse",
@@ -429,7 +264,9 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
429
264
  )
430
265
  pipeline_args["name"] = run_name
431
266
 
432
- if compute_target := self._create_or_get_compute(ml_client, settings):
267
+ if compute_target := create_or_get_compute(
268
+ ml_client, settings, default_compute_name=f"zenml_{self.id}"
269
+ ):
433
270
  pipeline_args["compute"] = compute_target
434
271
 
435
272
  @pipeline(force_rerun=True, **pipeline_args) # type: ignore[call-overload, misc]
@@ -540,5 +377,54 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
540
377
  )
541
378
 
542
379
  else:
543
- ml_client.jobs.create_or_update(pipeline_job)
380
+ job = ml_client.jobs.create_or_update(pipeline_job)
544
381
  logger.info(f"Pipeline {run_name} has been started.")
382
+ assert job.services is not None
383
+ assert job.name is not None
384
+
385
+ logger.info(
386
+ f"Pipeline {run_name} is running. "
387
+ "You can view the pipeline in the AzureML portal at "
388
+ f"{job.services['Studio'].endpoint}"
389
+ )
390
+
391
+ if self.config.synchronous:
392
+ logger.info("Waiting for pipeline to finish...")
393
+ ml_client.jobs.stream(job.name)
394
+
395
+ def get_pipeline_run_metadata(
396
+ self, run_id: UUID
397
+ ) -> Dict[str, "MetadataType"]:
398
+ """Get general component-specific metadata for a pipeline run.
399
+
400
+ Args:
401
+ run_id: The ID of the pipeline run.
402
+
403
+ Returns:
404
+ A dictionary of metadata.
405
+ """
406
+ try:
407
+ if connector := self.get_connector():
408
+ credentials = connector.connect()
409
+ else:
410
+ credentials = DefaultAzureCredential()
411
+
412
+ ml_client = MLClient(
413
+ credential=credentials,
414
+ subscription_id=self.config.subscription_id,
415
+ resource_group_name=self.config.resource_group,
416
+ workspace_name=self.config.workspace,
417
+ )
418
+
419
+ azureml_root_run_id = os.environ[ENV_ZENML_AZUREML_RUN_ID]
420
+ azureml_job = ml_client.jobs.get(azureml_root_run_id)
421
+
422
+ return {
423
+ METADATA_ORCHESTRATOR_URL: Uri(azureml_job.studio_url),
424
+ }
425
+ except Exception as e:
426
+ logger.warning(
427
+ f"Failed to fetch the Studio URL of the AzureML pipeline "
428
+ f"job: {e}"
429
+ )
430
+ return {}
@@ -69,6 +69,7 @@ class AzureMLEntrypointConfiguration(StepEntrypointConfiguration):
69
69
 
70
70
  # Azure automatically changes the working directory, we have to set it
71
71
  # back to /app before running the step.
72
+ os.makedirs("/app", exist_ok=True)
72
73
  os.chdir("/app")
73
74
 
74
75
  # Run the step
@@ -14,6 +14,7 @@
14
14
  """Azure Service Connector."""
15
15
 
16
16
  import datetime
17
+ import logging
17
18
  import re
18
19
  import subprocess
19
20
  from typing import Any, Dict, List, Optional, Tuple
@@ -58,6 +59,9 @@ from zenml.service_connectors.service_connector import (
58
59
  from zenml.utils.enum_utils import StrEnum
59
60
  from zenml.utils.secret_utils import PlainSerializedSecretStr
60
61
 
62
+ # Configure the logging level for azure.identity
63
+ logging.getLogger("azure.identity").setLevel(logging.WARNING)
64
+
61
65
  logger = get_logger(__name__)
62
66
 
63
67
 
@@ -13,29 +13,16 @@
13
13
  # permissions and limitations under the License.
14
14
  """Implementation of the ZenML AzureML Step Operator."""
15
15
 
16
- import itertools
17
- import os
18
16
  from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, cast
19
17
 
20
- from azureml.core import (
21
- ComputeTarget,
22
- Environment,
23
- Experiment,
24
- ScriptRunConfig,
25
- Workspace,
26
- )
27
- from azureml.core.authentication import (
28
- AbstractAuthentication,
29
- ServicePrincipalAuthentication,
30
- )
31
- from azureml.core.conda_dependencies import CondaDependencies
18
+ from azure.ai.ml import MLClient, command
19
+ from azure.ai.ml.entities import Environment
20
+ from azure.core.credentials import TokenCredential
21
+ from azure.identity import ClientSecretCredential, DefaultAzureCredential
32
22
 
33
- import zenml
34
- from zenml.client import Client
35
- from zenml.constants import (
36
- ENV_ZENML_CONFIG_PATH,
37
- )
38
- from zenml.environment import Environment as ZenMLEnvironment
23
+ from zenml.config.build_configuration import BuildConfiguration
24
+ from zenml.enums import StackComponentType
25
+ from zenml.integrations.azure.azureml_utils import create_or_get_compute
39
26
  from zenml.integrations.azure.flavors.azureml_step_operator_flavor import (
40
27
  AzureMLStepOperatorConfig,
41
28
  AzureMLStepOperatorSettings,
@@ -43,20 +30,18 @@ from zenml.integrations.azure.flavors.azureml_step_operator_flavor import (
43
30
  from zenml.logger import get_logger
44
31
  from zenml.stack import Stack, StackValidator
45
32
  from zenml.step_operators import BaseStepOperator
46
- from zenml.utils import source_utils
47
- from zenml.utils.pipeline_docker_image_builder import (
48
- DOCKER_IMAGE_ZENML_CONFIG_DIR,
49
- PipelineDockerImageBuilder,
50
- )
51
33
 
52
34
  if TYPE_CHECKING:
53
- from zenml.config import DockerSettings
54
35
  from zenml.config.base_settings import BaseSettings
55
36
  from zenml.config.step_run_info import StepRunInfo
37
+ from zenml.models import PipelineDeploymentBase
56
38
 
57
39
  logger = get_logger(__name__)
58
40
 
59
41
 
42
+ AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY = "azureml_step_operator"
43
+
44
+
60
45
  class AzureMLStepOperator(BaseStepOperator):
61
46
  """Step operator to run a step on AzureML.
62
47
 
@@ -87,11 +72,11 @@ class AzureMLStepOperator(BaseStepOperator):
87
72
  """Validates the stack.
88
73
 
89
74
  Returns:
90
- A validator that checks that the stack contains a remote artifact
91
- store.
75
+ A validator that checks that the stack contains a remote container
76
+ registry and a remote artifact store.
92
77
  """
93
78
 
94
- def _validate_remote_artifact_store(
79
+ def _validate_remote_components(
95
80
  stack: "Stack",
96
81
  ) -> Tuple[bool, str]:
97
82
  if stack.artifact_store.config.is_local:
@@ -104,113 +89,75 @@ class AzureMLStepOperator(BaseStepOperator):
104
89
  "step operator."
105
90
  )
106
91
 
92
+ container_registry = stack.container_registry
93
+ assert container_registry is not None
94
+
95
+ if container_registry.config.is_local:
96
+ return False, (
97
+ "The AzureML step operator runs code remotely and "
98
+ "needs to push/pull Docker images, but the "
99
+ f"container registry `{container_registry.name}` of the "
100
+ "active stack is local. Please ensure that your stack "
101
+ "contains a remote container registry when using the "
102
+ "AzureML step operator."
103
+ )
104
+
107
105
  return True, ""
108
106
 
109
107
  return StackValidator(
110
- custom_validation_function=_validate_remote_artifact_store,
108
+ required_components={
109
+ StackComponentType.CONTAINER_REGISTRY,
110
+ StackComponentType.IMAGE_BUILDER,
111
+ },
112
+ custom_validation_function=_validate_remote_components,
111
113
  )
112
114
 
113
- def _get_authentication(self) -> Optional[AbstractAuthentication]:
115
+ def _get_credentials(self) -> TokenCredential:
114
116
  """Returns the authentication object for the AzureML environment.
115
117
 
116
118
  Returns:
117
119
  The authentication object for the AzureML environment.
118
120
  """
119
- if (
121
+ # Authentication
122
+ if connector := self.get_connector():
123
+ credentials = connector.connect()
124
+ assert isinstance(credentials, TokenCredential)
125
+ return credentials
126
+ elif (
120
127
  self.config.tenant_id
121
128
  and self.config.service_principal_id
122
129
  and self.config.service_principal_password
123
130
  ):
124
- return ServicePrincipalAuthentication(
131
+ return ClientSecretCredential(
125
132
  tenant_id=self.config.tenant_id,
126
- service_principal_id=self.config.service_principal_id,
127
- service_principal_password=self.config.service_principal_password,
133
+ client_id=self.config.service_principal_id,
134
+ client_secret=self.config.service_principal_password,
128
135
  )
129
- return None
136
+ else:
137
+ return DefaultAzureCredential()
130
138
 
131
- def _prepare_environment(
132
- self,
133
- workspace: Workspace,
134
- docker_settings: "DockerSettings",
135
- run_name: str,
136
- environment_variables: Dict[str, str],
137
- environment_name: Optional[str] = None,
138
- ) -> Environment:
139
- """Prepares the environment in which Azure will run all jobs.
139
+ def get_docker_builds(
140
+ self, deployment: "PipelineDeploymentBase"
141
+ ) -> List["BuildConfiguration"]:
142
+ """Gets the Docker builds required for the component.
140
143
 
141
144
  Args:
142
- workspace: The AzureML Workspace that has configuration
143
- for a storage account, container registry among other
144
- things.
145
- docker_settings: The Docker settings for this step.
146
- run_name: The name of the pipeline run that can be used
147
- for naming environments and runs.
148
- environment_variables: Environment variables to set in the
149
- environment.
150
- environment_name: Optional name of an existing environment to use.
145
+ deployment: The pipeline deployment for which to get the builds.
151
146
 
152
147
  Returns:
153
- The AzureML Environment object.
148
+ The required Docker builds.
154
149
  """
155
- docker_image_builder = PipelineDockerImageBuilder()
156
- requirements_files = docker_image_builder.gather_requirements_files(
157
- docker_settings=docker_settings,
158
- stack=Client().active_stack,
159
- log=False,
160
- )
161
- requirements = list(
162
- itertools.chain.from_iterable(
163
- r[1].split("\n") for r in requirements_files
164
- )
165
- )
166
- requirements.append(f"zenml=={zenml.__version__}")
167
- logger.info(
168
- "Using requirements for AzureML step operator environment: %s",
169
- requirements,
170
- )
171
- if environment_name:
172
- environment = Environment.get(
173
- workspace=workspace, name=environment_name
174
- )
175
- if not environment.python.conda_dependencies:
176
- environment.python.conda_dependencies = (
177
- CondaDependencies.create(
178
- python_version=ZenMLEnvironment.python_version()
179
- )
150
+ builds = []
151
+ for step_name, step in deployment.step_configurations.items():
152
+ if step.config.step_operator == self.name:
153
+ build = BuildConfiguration(
154
+ key=AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY,
155
+ settings=step.config.docker_settings,
156
+ step_name=step_name,
180
157
  )
158
+ builds.append(build)
181
159
 
182
- for requirement in requirements:
183
- environment.python.conda_dependencies.add_pip_package(
184
- requirement
185
- )
186
- else:
187
- environment = Environment(name=f"zenml-{run_name}")
188
- environment.python.conda_dependencies = CondaDependencies.create(
189
- pip_packages=requirements,
190
- python_version=ZenMLEnvironment.python_version(),
191
- )
192
-
193
- if docker_settings.parent_image:
194
- # replace the default azure base image
195
- environment.docker.base_image = docker_settings.parent_image
196
-
197
- # set credentials to access azure storage
198
- for key in [
199
- "AZURE_STORAGE_ACCOUNT_KEY",
200
- "AZURE_STORAGE_ACCOUNT_NAME",
201
- "AZURE_STORAGE_CONNECTION_STRING",
202
- "AZURE_STORAGE_SAS_TOKEN",
203
- ]:
204
- value = os.getenv(key)
205
- if value:
206
- environment_variables[key] = value
207
-
208
- environment_variables[ENV_ZENML_CONFIG_PATH] = (
209
- f"./{DOCKER_IMAGE_ZENML_CONFIG_DIR}"
210
- )
211
- environment_variables.update(docker_settings.environment)
212
- environment.environment_variables = environment_variables
213
- return environment
160
+ return builds
214
161
 
215
162
  def launch(
216
163
  self,
@@ -226,79 +173,33 @@ class AzureMLStepOperator(BaseStepOperator):
226
173
  environment: Environment variables to set in the step operator
227
174
  environment.
228
175
  """
229
- if not info.config.resource_settings.empty:
230
- logger.warning(
231
- "Specifying custom step resources is not supported for "
232
- "the AzureML step operator. If you want to run this step "
233
- "operator on specific resources, you can do so by creating an "
234
- "Azure compute target (https://docs.microsoft.com/en-us/azure/machine-learning/concept-compute-target) "
235
- "with a specific machine type and then updating this step "
236
- "operator: `zenml step-operator update %s "
237
- "--compute_target_name=<COMPUTE_TARGET_NAME>`",
238
- self.name,
239
- )
240
-
241
- unused_docker_fields = [
242
- "dockerfile",
243
- "build_context_root",
244
- "build_options",
245
- "skip_build",
246
- "target_repository",
247
- "dockerignore",
248
- "copy_files",
249
- "copy_global_config",
250
- "apt_packages",
251
- "user",
252
- "source_files",
253
- "allow_including_files_in_images",
254
- "allow_download_from_code_repository",
255
- "allow_download_from_artifact_store",
256
- ]
257
- docker_settings = info.config.docker_settings
258
- ignored_docker_fields = docker_settings.model_fields_set.intersection(
259
- unused_docker_fields
260
- )
261
-
262
- if ignored_docker_fields:
263
- logger.warning(
264
- "The AzureML step operator currently does not support all "
265
- "options defined in your Docker settings. Ignoring all "
266
- "values set for the attributes: %s",
267
- ignored_docker_fields,
268
- )
269
-
270
176
  settings = cast(AzureMLStepOperatorSettings, self.get_settings(info))
177
+ image_name = info.get_image(key=AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY)
271
178
 
272
- workspace = Workspace.get(
179
+ # Client creation
180
+ ml_client = MLClient(
181
+ credential=self._get_credentials(),
273
182
  subscription_id=self.config.subscription_id,
274
- resource_group=self.config.resource_group,
275
- name=self.config.workspace_name,
276
- auth=self._get_authentication(),
183
+ resource_group_name=self.config.resource_group,
184
+ workspace_name=self.config.workspace_name,
277
185
  )
278
186
 
279
- source_directory = source_utils.get_source_root()
187
+ env = Environment(name=f"zenml-{info.run_name}", image=image_name)
280
188
 
281
- environment = self._prepare_environment(
282
- workspace=workspace,
283
- docker_settings=docker_settings,
284
- run_name=info.run_name,
285
- environment_variables=environment,
286
- environment_name=settings.environment_name,
287
- )
288
- compute_target = ComputeTarget(
289
- workspace=workspace, name=self.config.compute_target_name
189
+ compute_target = create_or_get_compute(
190
+ ml_client, settings, default_compute_name=f"zenml_{self.id}"
290
191
  )
291
192
 
292
- run_config = ScriptRunConfig(
293
- source_directory=source_directory,
294
- environment=environment,
295
- compute_target=compute_target,
296
- command=entrypoint_command,
193
+ command_job = command(
194
+ name=info.run_name,
195
+ command=" ".join(entrypoint_command),
196
+ environment=env,
197
+ environment_variables=environment,
198
+ compute=compute_target,
199
+ experiment_name=info.pipeline.name,
297
200
  )
298
201
 
299
- experiment = Experiment(workspace=workspace, name=info.pipeline.name)
300
- run = experiment.submit(config=run_config)
202
+ job = ml_client.jobs.create_or_update(command_job)
301
203
 
302
- run.display_name = info.run_name
303
- info.force_write_logs()
304
- run.wait_for_completion(show_output=True)
204
+ logger.info(f"AzureML job created with id: {job.id}")
205
+ ml_client.jobs.stream(info.run_name)
@@ -44,7 +44,9 @@ LIGHTGBM = "lightgbm"
44
44
  MLFLOW = "mlflow"
45
45
  NEPTUNE = "neptune"
46
46
  NEURAL_PROPHET = "neural_prophet"
47
+ NUMPY = "numpy"
47
48
  OPEN_AI = "openai"
49
+ PANDAS = "pandas"
48
50
  PIGEON = "pigeon"
49
51
  PILLOW = "pillow"
50
52
  PLOTLY = "plotly"
@@ -73,3 +75,4 @@ WANDB = "wandb"
73
75
  VERTEX = "vertex"
74
76
  XGBOOST = "xgboost"
75
77
  VAULT = "vault"
78
+ LIGHTNING = "lightning"