zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -381,6 +381,36 @@ class ServerConfiguration(BaseModel):
381
381
  # Revert to the default value if the header is enabled
382
382
  del data[k]
383
383
 
384
+ # Handle merging of user-defined secure_headers_csp value with the default value
385
+ if "secure_headers_csp" in data:
386
+ user_defined_csp = data["secure_headers_csp"]
387
+ if isinstance(user_defined_csp, str):
388
+ # Parse the user-defined CSP string into a dictionary
389
+ user_defined_csp_dict = {}
390
+ for directive in user_defined_csp.split(";"):
391
+ directive = directive.strip()
392
+ if directive:
393
+ key, value = directive.split(" ", 1)
394
+ user_defined_csp_dict[key] = value.strip("'\"")
395
+
396
+ # Merge the user-defined CSP dictionary with the default CSP dictionary
397
+ default_csp_dict = {}
398
+ for directive in DEFAULT_ZENML_SERVER_SECURE_HEADERS_CSP.split(
399
+ ";"
400
+ ):
401
+ directive = directive.strip()
402
+ if directive:
403
+ key, value = directive.split(" ", 1)
404
+ default_csp_dict[key] = value.strip("'\"")
405
+
406
+ merged_csp_dict = {**default_csp_dict, **user_defined_csp_dict}
407
+
408
+ # Convert the merged CSP dictionary back to a string
409
+ merged_csp_str = "; ".join(
410
+ f"{key} {value}" for key, value in merged_csp_dict.items()
411
+ )
412
+ data["secure_headers_csp"] = merged_csp_str
413
+
384
414
  return data
385
415
 
386
416
  @property
zenml/config/source.py CHANGED
@@ -234,20 +234,16 @@ class NotebookSource(Source):
234
234
  """Source representing an object defined in a notebook.
235
235
 
236
236
  Attributes:
237
- code_path: Path where the notebook cell code for this source is
238
- uploaded.
239
237
  replacement_module: Name of the module from which this source should
240
238
  be loaded in case the code is not running in a notebook.
239
+ artifact_store_id: ID of the artifact store in which the replacement
240
+ module code is stored.
241
241
  """
242
242
 
243
- code_path: Optional[str] = None
244
243
  replacement_module: Optional[str] = None
244
+ artifact_store_id: Optional[UUID] = None
245
245
  type: SourceType = SourceType.NOTEBOOK
246
246
 
247
- # Private attribute that is used to store the code but should not be
248
- # serialized
249
- _cell_code: Optional[str] = None
250
-
251
247
  @field_validator("type")
252
248
  @classmethod
253
249
  def _validate_type(cls, value: SourceType) -> SourceType:
zenml/constants.py CHANGED
@@ -162,7 +162,6 @@ ENV_ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING = (
162
162
  )
163
163
  ENV_ZENML_DISABLE_WORKSPACE_WARNINGS = "ZENML_DISABLE_WORKSPACE_WARNINGS"
164
164
  ENV_ZENML_SKIP_IMAGE_BUILDER_DEFAULT = "ZENML_SKIP_IMAGE_BUILDER_DEFAULT"
165
- ENV_ZENML_REQUIRES_CODE_DOWNLOAD = "ZENML_REQUIRES_CODE_DOWNLOAD"
166
165
  ENV_ZENML_SERVER = "ZENML_SERVER"
167
166
  ENV_ZENML_LOCAL_SERVER = "ZENML_LOCAL_SERVER"
168
167
  ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
@@ -173,6 +172,7 @@ ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
173
172
  )
174
173
  ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
175
174
  ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
175
+ ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
176
176
 
177
177
  # ZenML Server environment variables
178
178
  ENV_ZENML_SERVER_PREFIX = "ZENML_SERVER_"
@@ -185,6 +185,9 @@ ENV_ZENML_SERVER_USE_LEGACY_DASHBOARD = (
185
185
  f"{ENV_ZENML_SERVER_PREFIX}USE_LEGACY_DASHBOARD"
186
186
  )
187
187
  ENV_ZENML_SERVER_AUTO_ACTIVATE = f"{ENV_ZENML_SERVER_PREFIX}AUTO_ACTIVATE"
188
+ ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK = (
189
+ "ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK"
190
+ )
188
191
 
189
192
  # Logging variables
190
193
  IS_DEBUG_ENV: bool = handle_bool_env_var(ENV_ZENML_DEBUG, default=False)
@@ -348,7 +351,6 @@ EMAIL_ANALYTICS = "/email-opt-in"
348
351
  EVENT_FLAVORS = "/event-flavors"
349
352
  EVENT_SOURCES = "/event-sources"
350
353
  FLAVORS = "/flavors"
351
- FULL_STACK = "/full-stack"
352
354
  GET_OR_CREATE = "/get-or-create"
353
355
  GRAPH = "/graph"
354
356
  HEALTH = "/health"
@@ -486,7 +488,7 @@ MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION = (
486
488
  )
487
489
 
488
490
 
489
- FINISHED_ONBOARDING_SURVEY_KEY = "awareness_channels"
491
+ FINISHED_ONBOARDING_SURVEY_KEY = "finished_onboarding_survey"
490
492
 
491
493
  # Name validation
492
494
  BANNED_NAME_CHARACTERS = "\t\n\r\v\f"
@@ -17,15 +17,11 @@ import argparse
17
17
  import os
18
18
  import sys
19
19
  from abc import ABC, abstractmethod
20
- from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Set
20
+ from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Optional, Set
21
21
  from uuid import UUID
22
22
 
23
23
  from zenml.client import Client
24
24
  from zenml.code_repositories import BaseCodeRepository
25
- from zenml.constants import (
26
- ENV_ZENML_REQUIRES_CODE_DOWNLOAD,
27
- handle_bool_env_var,
28
- )
29
25
  from zenml.logger import get_logger
30
26
  from zenml.utils import (
31
27
  code_repository_utils,
@@ -194,22 +190,28 @@ class BaseEntrypointConfiguration(ABC):
194
190
  return Client().zen_store.get_deployment(deployment_id=deployment_id)
195
191
 
196
192
  def download_code_if_necessary(
197
- self, deployment: "PipelineDeploymentResponse"
193
+ self,
194
+ deployment: "PipelineDeploymentResponse",
195
+ step_name: Optional[str] = None,
198
196
  ) -> None:
199
197
  """Downloads user code if necessary.
200
198
 
201
199
  Args:
202
200
  deployment: The deployment for which to download the code.
201
+ step_name: Name of the step to be run. This will be used to
202
+ determine whether code download is necessary. If not given,
203
+ the DockerSettings of the pipeline will be used to make that
204
+ decision instead.
203
205
 
204
206
  Raises:
205
207
  RuntimeError: If the current environment requires code download
206
208
  but the deployment does not have a reference to any code.
207
209
  """
208
- requires_code_download = handle_bool_env_var(
209
- ENV_ZENML_REQUIRES_CODE_DOWNLOAD
210
+ should_download_code = self._should_download_code(
211
+ deployment=deployment, step_name=step_name
210
212
  )
211
213
 
212
- if not requires_code_download:
214
+ if not should_download_code:
213
215
  return
214
216
 
215
217
  if code_reference := deployment.code_reference:
@@ -217,7 +219,7 @@ class BaseEntrypointConfiguration(ABC):
217
219
  code_reference=code_reference
218
220
  )
219
221
  elif code_path := deployment.code_path:
220
- self.download_code_from_artifact_store(code_path=code_path)
222
+ code_utils.download_code_from_artifact_store(code_path=code_path)
221
223
  else:
222
224
  raise RuntimeError(
223
225
  "Code download required but no code reference or path provided."
@@ -259,30 +261,42 @@ class BaseEntrypointConfiguration(ABC):
259
261
  sys.path.insert(0, download_dir)
260
262
  os.chdir(download_dir)
261
263
 
262
- def download_code_from_artifact_store(self, code_path: str) -> None:
263
- """Download code from the artifact store.
264
+ def _should_download_code(
265
+ self,
266
+ deployment: "PipelineDeploymentResponse",
267
+ step_name: Optional[str] = None,
268
+ ) -> bool:
269
+ """Checks whether code should be downloaded.
264
270
 
265
271
  Args:
266
- code_path: Path where the code is stored.
272
+ deployment: The deployment to check.
273
+ step_name: Name of the step to be run. This will be used to
274
+ determine whether code download is necessary. If not given,
275
+ the DockerSettings of the pipeline will be used to make that
276
+ decision instead.
277
+
278
+ Returns:
279
+ Whether code should be downloaded.
267
280
  """
268
- logger.info(
269
- "Downloading code from artifact store path `%s`.", code_path
281
+ docker_settings = (
282
+ deployment.step_configurations[step_name].config.docker_settings
283
+ if step_name
284
+ else deployment.pipeline_configuration.docker_settings
270
285
  )
271
286
 
272
- # Do not remove this line, we need to instantiate the artifact store to
273
- # register the filesystem needed for the file download
274
- _ = Client().active_stack.artifact_store
275
-
276
- extract_dir = os.path.abspath("code")
277
- os.makedirs(extract_dir)
287
+ if (
288
+ deployment.code_reference
289
+ and docker_settings.allow_download_from_code_repository
290
+ ):
291
+ return True
278
292
 
279
- code_utils.download_and_extract_code(
280
- code_path=code_path, extract_dir=extract_dir
281
- )
293
+ if (
294
+ deployment.code_path
295
+ and docker_settings.allow_download_from_artifact_store
296
+ ):
297
+ return True
282
298
 
283
- source_utils.set_custom_source_root(extract_dir)
284
- sys.path.insert(0, extract_dir)
285
- os.chdir(extract_dir)
299
+ return False
286
300
 
287
301
  @abstractmethod
288
302
  def run(self) -> None:
@@ -151,9 +151,12 @@ class StepEntrypointConfiguration(BaseEntrypointConfiguration):
151
151
  # and stack component flavors are registered.
152
152
  integration_registry.activate_integrations()
153
153
 
154
- self.download_code_if_necessary(deployment=deployment)
155
-
156
154
  step_name = self.entrypoint_args[STEP_NAME_OPTION]
155
+
156
+ self.download_code_if_necessary(
157
+ deployment=deployment, step_name=step_name
158
+ )
159
+
157
160
  pipeline_name = deployment.pipeline_configuration.name
158
161
 
159
162
  step = deployment.step_configurations[step_name]
zenml/enums.py CHANGED
@@ -303,6 +303,8 @@ class EnvironmentType(StrEnum):
303
303
  PAPERSPACE = "paperspace"
304
304
  WSL = "wsl"
305
305
  LIGHTNING_AI_STUDIO = "lightning_ai_studio"
306
+ GITHUB_CODESPACES = "github_codespaces"
307
+ VSCODE_REMOTE_CONTAINER = "vscode_remote_container"
306
308
 
307
309
 
308
310
  class ModelStages(StrEnum):
zenml/environment.py CHANGED
@@ -69,6 +69,10 @@ def get_environment() -> str:
69
69
  return EnvironmentType.BITBUCKET_CI
70
70
  elif Environment.in_ci():
71
71
  return EnvironmentType.GENERIC_CI
72
+ elif Environment.in_github_codespaces():
73
+ return EnvironmentType.GITHUB_CODESPACES
74
+ elif Environment.in_vscode_remote_container():
75
+ return EnvironmentType.VSCODE_REMOTE_CONTAINER
72
76
  elif Environment.in_lightning_ai_studio():
73
77
  return EnvironmentType.LIGHTNING_AI_STUDIO
74
78
  elif Environment.in_docker():
@@ -271,6 +275,33 @@ class Environment(metaclass=SingletonMetaClass):
271
275
  return True
272
276
  return False
273
277
 
278
+ @staticmethod
279
+ def in_github_codespaces() -> bool:
280
+ """If the current Python process is running in GitHub Codespaces.
281
+
282
+ Returns:
283
+ `True` if the current Python process is running in GitHub Codespaces,
284
+ `False` otherwise.
285
+ """
286
+ return (
287
+ "CODESPACES" in os.environ
288
+ or "GITHUB_CODESPACE_TOKEN" in os.environ
289
+ or "GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" in os.environ
290
+ )
291
+
292
+ @staticmethod
293
+ def in_vscode_remote_container() -> bool:
294
+ """If the current Python process is running in a VS Code Remote Container.
295
+
296
+ Returns:
297
+ `True` if the current Python process is running in a VS Code Remote Container,
298
+ `False` otherwise.
299
+ """
300
+ return (
301
+ "REMOTE_CONTAINERS" in os.environ
302
+ or "VSCODE_REMOTE_CONTAINERS_SESSION" in os.environ
303
+ )
304
+
274
305
  @staticmethod
275
306
  def in_paperspace_gradient() -> bool:
276
307
  """If the current Python process is running in Paperspace Gradient.
@@ -14,9 +14,7 @@
14
14
  """The base class for feature stores."""
15
15
 
16
16
  from abc import ABC, abstractmethod
17
- from typing import Any, Dict, List, Type, Union, cast
18
-
19
- import pandas as pd
17
+ from typing import Any, Dict, List, Type, cast
20
18
 
21
19
  from zenml.enums import StackComponentType
22
20
  from zenml.stack import Flavor, StackComponent
@@ -42,10 +40,10 @@ class BaseFeatureStore(StackComponent, ABC):
42
40
  @abstractmethod
43
41
  def get_historical_features(
44
42
  self,
45
- entity_df: Union[pd.DataFrame, str],
43
+ entity_df: Any,
46
44
  features: List[str],
47
45
  full_feature_names: bool = False,
48
- ) -> pd.DataFrame:
46
+ ) -> Any:
49
47
  """Returns the historical features for training or batch scoring.
50
48
 
51
49
  Args:
@@ -54,7 +52,7 @@ class BaseFeatureStore(StackComponent, ABC):
54
52
  full_feature_names: Whether to return the full feature names.
55
53
 
56
54
  Returns:
57
- The historical features as a Pandas DataFrame.
55
+ The historical features.
58
56
  """
59
57
 
60
58
  @abstractmethod
@@ -36,6 +36,7 @@ from zenml.integrations.gitlab import GitLabIntegration # noqa
36
36
  from zenml.integrations.great_expectations import ( # noqa
37
37
  GreatExpectationsIntegration,
38
38
  )
39
+ from zenml.integrations.lightning import LightningIntegration # noqa
39
40
  from zenml.integrations.huggingface import HuggingfaceIntegration # noqa
40
41
  from zenml.integrations.hyperai import HyperAIIntegration # noqa
41
42
  from zenml.integrations.kaniko import KanikoIntegration # noqa
@@ -48,7 +49,9 @@ from zenml.integrations.lightgbm import LightGBMIntegration # noqa
48
49
  from zenml.integrations.mlflow import MlflowIntegration # noqa
49
50
  from zenml.integrations.neptune import NeptuneIntegration # noqa
50
51
  from zenml.integrations.neural_prophet import NeuralProphetIntegration # noqa
52
+ from zenml.integrations.numpy import NumpyIntegration # noqa
51
53
  from zenml.integrations.openai import OpenAIIntegration # noqa
54
+ from zenml.integrations.pandas import PandasIntegration # noqa
52
55
  from zenml.integrations.pigeon import PigeonIntegration # noqa
53
56
  from zenml.integrations.pillow import PillowIntegration # noqa
54
57
  from zenml.integrations.polars import PolarsIntegration # noqa
@@ -119,6 +119,15 @@ class AirflowOrchestratorConfig(
119
119
 
120
120
  local: bool = True
121
121
 
122
+ @property
123
+ def is_schedulable(self) -> bool:
124
+ """Whether the orchestrator is schedulable or not.
125
+
126
+ Returns:
127
+ Whether the orchestrator is schedulable or not.
128
+ """
129
+ return True
130
+
122
131
 
123
132
  class AirflowOrchestratorFlavor(BaseOrchestratorFlavor):
124
133
  """Flavor for the Airflow orchestrator."""
@@ -45,8 +45,8 @@ class AWSIntegration(Integration):
45
45
  ]
46
46
  REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
47
47
 
48
- @staticmethod
49
- def activate() -> None:
48
+ @classmethod
49
+ def activate(cls) -> None:
50
50
  """Activate the AWS integration."""
51
51
  from zenml.integrations.aws import service_connectors # noqa
52
52
 
@@ -372,7 +372,8 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
372
372
  # mainly for testing purposes, we wait for the pipeline to finish
373
373
  if self.config.synchronous:
374
374
  logger.info(
375
- "Executing synchronously. Waiting for pipeline to finish..."
375
+ "Executing synchronously. Waiting for pipeline to finish... \n"
376
+ "At this point you can `Ctrl-C` out without cancelling the execution."
376
377
  )
377
378
  try:
378
379
  pipeline_execution.wait(
@@ -52,8 +52,8 @@ class AzureIntegration(Integration):
52
52
  ]
53
53
  REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
54
54
 
55
- @staticmethod
56
- def activate() -> None:
55
+ @classmethod
56
+ def activate(cls) -> None:
57
57
  """Activate the Azure integration."""
58
58
  from zenml.integrations.azure import service_connectors # noqa
59
59
 
@@ -0,0 +1,201 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """AzureML definitions."""
15
+
16
+ from typing import Optional
17
+
18
+ from azure.ai.ml import MLClient
19
+ from azure.ai.ml.entities import Compute
20
+ from azure.core.exceptions import (
21
+ ResourceNotFoundError,
22
+ )
23
+
24
+ from zenml.integrations.azure.flavors.azureml import (
25
+ AzureMLComputeSettings,
26
+ AzureMLComputeTypes,
27
+ )
28
+ from zenml.logger import get_logger
29
+
30
+ logger = get_logger(__name__)
31
+
32
+
33
+ def check_settings_and_compute_configuration(
34
+ parameter: str,
35
+ settings: AzureMLComputeSettings,
36
+ compute: Compute,
37
+ ) -> None:
38
+ """Utility function comparing a parameter between settings and compute.
39
+
40
+ Args:
41
+ parameter: the name of the parameter.
42
+ settings: The AzureML orchestrator settings.
43
+ compute: The compute instance or cluster from AzureML.
44
+ """
45
+ # Check the compute size
46
+ compute_value = getattr(compute, parameter)
47
+ settings_value = getattr(settings, parameter)
48
+
49
+ if settings_value is not None and settings_value != compute_value:
50
+ logger.warning(
51
+ f"The '{parameter}' defined in the settings '{settings_value}' "
52
+ "does not match the actual parameter of the instance: "
53
+ f"'{compute_value}'. Will ignore this setting for now."
54
+ )
55
+
56
+
57
+ def create_or_get_compute(
58
+ client: MLClient,
59
+ settings: AzureMLComputeSettings,
60
+ default_compute_name: str,
61
+ ) -> Optional[str]:
62
+ """Creates or fetches the compute target if defined in the settings.
63
+
64
+ Args:
65
+ client: the AzureML client.
66
+ settings: the settings for the orchestrator.
67
+ default_compute_name: the default name for the compute target, if one
68
+ is not provided in the settings.
69
+
70
+ Returns:
71
+ None, if the orchestrator is using serverless compute or
72
+ str, the name of the compute target (instance or cluster).
73
+
74
+ Raises:
75
+ RuntimeError: if the fetched compute target is unsupported or the
76
+ mode defined in the setting does not match the type of the
77
+ compute target.
78
+ """
79
+ # If the mode is serverless, we can not fetch anything anyhow
80
+ if settings.mode == AzureMLComputeTypes.SERVERLESS:
81
+ return None
82
+
83
+ # If a name is not provided, generate one based on the orchestrator id
84
+ compute_name = settings.compute_name or default_compute_name
85
+
86
+ # Try to fetch the compute target
87
+ try:
88
+ compute = client.compute.get(compute_name)
89
+
90
+ logger.info(f"Using existing compute target: '{compute_name}'.")
91
+
92
+ # Check if compute size matches with the settings
93
+ check_settings_and_compute_configuration(
94
+ parameter="size", settings=settings, compute=compute
95
+ )
96
+
97
+ compute_type = compute.type
98
+
99
+ # Check the type and matches the settings
100
+ if compute_type == "computeinstance": # Compute Instance
101
+ if settings.mode != AzureMLComputeTypes.COMPUTE_INSTANCE:
102
+ raise RuntimeError(
103
+ "The mode of operation for the compute target defined"
104
+ f"in the settings '{settings.mode}' does not match "
105
+ f"the type of the compute target: `{compute_name}` "
106
+ "which is a 'compute-instance'. Please make sure that "
107
+ "the settings are adjusted properly."
108
+ )
109
+
110
+ if compute.state != "Running":
111
+ raise RuntimeError(
112
+ f"The compute instance `{compute_name}` is not in a "
113
+ "running state at the moment. Please make sure that "
114
+ "the compute target is running, before executing the "
115
+ "pipeline."
116
+ )
117
+
118
+ # Idle time before shutdown
119
+ check_settings_and_compute_configuration(
120
+ parameter="idle_time_before_shutdown_minutes",
121
+ settings=settings,
122
+ compute=compute,
123
+ )
124
+
125
+ elif compute_type == "amIcompute": # Compute Cluster
126
+ if settings.mode != AzureMLComputeTypes.COMPUTE_CLUSTER:
127
+ raise RuntimeError(
128
+ "The mode of operation for the compute target defined "
129
+ f"in the settings '{settings.mode}' does not match "
130
+ f"the type of the compute target: `{compute_name}` "
131
+ "which is a 'compute-cluster'. Please make sure that "
132
+ "the settings are adjusted properly."
133
+ )
134
+
135
+ if compute.provisioning_state != "Succeeded":
136
+ raise RuntimeError(
137
+ f"The provisioning state '{compute.provisioning_state}'"
138
+ f"of the compute cluster `{compute_name}` is not "
139
+ "successful. Please make sure that the compute cluster "
140
+ "is provisioned properly, before executing the "
141
+ "pipeline."
142
+ )
143
+
144
+ for parameter in [
145
+ "idle_time_before_scale_down",
146
+ "max_instances",
147
+ "min_instances",
148
+ "tier",
149
+ "location",
150
+ ]:
151
+ # Check all possible configurations
152
+ check_settings_and_compute_configuration(
153
+ parameter=parameter, settings=settings, compute=compute
154
+ )
155
+ else:
156
+ raise RuntimeError(f"Unsupported compute type: {compute_type}")
157
+ return compute_name
158
+
159
+ # If the compute target does not exist create it
160
+ except ResourceNotFoundError:
161
+ logger.info(
162
+ "Can not find the compute target with name: " f"'{compute_name}':"
163
+ )
164
+
165
+ if settings.mode == AzureMLComputeTypes.COMPUTE_INSTANCE:
166
+ logger.info(
167
+ "Creating a new compute instance. This might take a "
168
+ "few minutes."
169
+ )
170
+
171
+ from azure.ai.ml.entities import ComputeInstance
172
+
173
+ compute_instance = ComputeInstance(
174
+ name=compute_name,
175
+ size=settings.size,
176
+ idle_time_before_shutdown_minutes=settings.idle_time_before_shutdown_minutes,
177
+ )
178
+ client.begin_create_or_update(compute_instance).result()
179
+ return compute_name
180
+
181
+ elif settings.mode == AzureMLComputeTypes.COMPUTE_CLUSTER:
182
+ logger.info(
183
+ "Creating a new compute cluster. This might take a "
184
+ "few minutes."
185
+ )
186
+
187
+ from azure.ai.ml.entities import AmlCompute
188
+
189
+ compute_cluster = AmlCompute(
190
+ name=compute_name,
191
+ size=settings.size,
192
+ location=settings.location,
193
+ min_instances=settings.min_instances,
194
+ max_instances=settings.max_instances,
195
+ idle_time_before_scale_down=settings.idle_time_before_scaledown_down,
196
+ tier=settings.tier,
197
+ )
198
+ client.begin_create_or_update(compute_cluster).result()
199
+ return compute_name
200
+
201
+ return None