zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -26,6 +26,7 @@ from typing import (
|
|
26
26
|
Union,
|
27
27
|
cast,
|
28
28
|
)
|
29
|
+
from uuid import UUID
|
29
30
|
|
30
31
|
from azure.ai.ml import Input, MLClient, Output
|
31
32
|
from azure.ai.ml.constants import TimeZone
|
@@ -40,15 +41,16 @@ from azure.ai.ml.entities import (
|
|
40
41
|
from azure.core.exceptions import (
|
41
42
|
HttpResponseError,
|
42
43
|
ResourceExistsError,
|
43
|
-
ResourceNotFoundError,
|
44
44
|
)
|
45
45
|
from azure.identity import DefaultAzureCredential
|
46
46
|
|
47
47
|
from zenml.config.base_settings import BaseSettings
|
48
48
|
from zenml.config.step_configurations import Step
|
49
|
+
from zenml.constants import METADATA_ORCHESTRATOR_URL
|
49
50
|
from zenml.enums import StackComponentType
|
51
|
+
from zenml.integrations.azure.azureml_utils import create_or_get_compute
|
52
|
+
from zenml.integrations.azure.flavors.azureml import AzureMLComputeTypes
|
50
53
|
from zenml.integrations.azure.flavors.azureml_orchestrator_flavor import (
|
51
|
-
AzureMLComputeTypes,
|
52
54
|
AzureMLOrchestratorConfig,
|
53
55
|
AzureMLOrchestratorSettings,
|
54
56
|
)
|
@@ -56,14 +58,13 @@ from zenml.integrations.azure.orchestrators.azureml_orchestrator_entrypoint_conf
|
|
56
58
|
AzureMLEntrypointConfiguration,
|
57
59
|
)
|
58
60
|
from zenml.logger import get_logger
|
61
|
+
from zenml.metadata.metadata_types import MetadataType, Uri
|
59
62
|
from zenml.orchestrators import ContainerizedOrchestrator
|
60
63
|
from zenml.orchestrators.utils import get_orchestrator_run_name
|
61
64
|
from zenml.stack import StackValidator
|
62
65
|
from zenml.utils.string_utils import b64_encode
|
63
66
|
|
64
67
|
if TYPE_CHECKING:
|
65
|
-
from azure.ai.ml.entities import AmlCompute, ComputeInstance
|
66
|
-
|
67
68
|
from zenml.models import PipelineDeploymentResponse
|
68
69
|
from zenml.stack import Stack
|
69
70
|
|
@@ -193,172 +194,6 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
|
|
193
194
|
command=" ".join(command + arguments),
|
194
195
|
)
|
195
196
|
|
196
|
-
@staticmethod
|
197
|
-
def _check_settings_and_compute_configuration(
|
198
|
-
parameter: str,
|
199
|
-
settings: AzureMLOrchestratorSettings,
|
200
|
-
compute: Union["ComputeInstance", "AmlCompute"],
|
201
|
-
) -> None:
|
202
|
-
"""Utility function comparing a parameter between settings and compute.
|
203
|
-
|
204
|
-
Args:
|
205
|
-
parameter: the name of the parameter.
|
206
|
-
settings: The AzureML orchestrator settings.
|
207
|
-
compute: The compute instance or cluster from AzureML.
|
208
|
-
"""
|
209
|
-
# Check the compute size
|
210
|
-
compute_value = getattr(compute, parameter)
|
211
|
-
settings_value = getattr(settings, parameter)
|
212
|
-
|
213
|
-
if settings_value is not None and settings_value != compute_value:
|
214
|
-
logger.warning(
|
215
|
-
f"The '{parameter}' defined in the settings '{settings_value}' "
|
216
|
-
"does not match the actual parameter of the instance: "
|
217
|
-
f"'{compute_value}'. Will ignore this setting for now."
|
218
|
-
)
|
219
|
-
|
220
|
-
def _create_or_get_compute(
|
221
|
-
self, client: MLClient, settings: AzureMLOrchestratorSettings
|
222
|
-
) -> Optional[str]:
|
223
|
-
"""Creates or fetches the compute target if defined in the settings.
|
224
|
-
|
225
|
-
Args:
|
226
|
-
client: the AzureML client.
|
227
|
-
settings: the settings for the orchestrator.
|
228
|
-
|
229
|
-
Returns:
|
230
|
-
None, if the orchestrator is using serverless compute or
|
231
|
-
str, the name of the compute target (instance or cluster).
|
232
|
-
|
233
|
-
Raises:
|
234
|
-
RuntimeError: if the fetched compute target is unsupported or the
|
235
|
-
mode defined in the setting does not match the type of the
|
236
|
-
compute target.
|
237
|
-
"""
|
238
|
-
# If the mode is serverless, we can not fetch anything anyhow
|
239
|
-
if settings.mode == AzureMLComputeTypes.SERVERLESS:
|
240
|
-
return None
|
241
|
-
|
242
|
-
# If a name is not provided, generate one based on the orchestrator id
|
243
|
-
compute_name = settings.compute_name or f"compute_{self.id}"
|
244
|
-
# Try to fetch the compute target
|
245
|
-
try:
|
246
|
-
compute = client.compute.get(compute_name)
|
247
|
-
|
248
|
-
logger.info(f"Using existing compute target: '{compute_name}'.")
|
249
|
-
|
250
|
-
# Check if compute size matches with the settings
|
251
|
-
self._check_settings_and_compute_configuration(
|
252
|
-
parameter="size", settings=settings, compute=compute
|
253
|
-
)
|
254
|
-
|
255
|
-
compute_type = compute.type
|
256
|
-
|
257
|
-
# Check the type and matches the settings
|
258
|
-
if compute_type == "computeinstance": # Compute Instance
|
259
|
-
if settings.mode != AzureMLComputeTypes.COMPUTE_INSTANCE:
|
260
|
-
raise RuntimeError(
|
261
|
-
"The mode of operation for the compute target defined"
|
262
|
-
f"in the settings '{settings.mode}' does not match "
|
263
|
-
f"the type of the compute target: `{compute_name}` "
|
264
|
-
"which is a 'compute-instance'. Please make sure that "
|
265
|
-
"the settings are adjusted properly."
|
266
|
-
)
|
267
|
-
|
268
|
-
if compute.state != "Running":
|
269
|
-
raise RuntimeError(
|
270
|
-
f"The compute instance `{compute_name}` is not in a "
|
271
|
-
"running state at the moment. Please make sure that "
|
272
|
-
"the compute target is running, before executing the "
|
273
|
-
"pipeline."
|
274
|
-
)
|
275
|
-
|
276
|
-
# Idle time before shutdown
|
277
|
-
self._check_settings_and_compute_configuration(
|
278
|
-
parameter="idle_time_before_shutdown_minutes",
|
279
|
-
settings=settings,
|
280
|
-
compute=compute,
|
281
|
-
)
|
282
|
-
|
283
|
-
elif compute_type == "amIcompute": # Compute Cluster
|
284
|
-
if settings.mode != AzureMLComputeTypes.COMPUTE_CLUSTER:
|
285
|
-
raise RuntimeError(
|
286
|
-
"The mode of operation for the compute target defined "
|
287
|
-
f"in the settings '{settings.mode}' does not match "
|
288
|
-
f"the type of the compute target: `{compute_name}` "
|
289
|
-
"which is a 'compute-cluster'. Please make sure that "
|
290
|
-
"the settings are adjusted properly."
|
291
|
-
)
|
292
|
-
|
293
|
-
if compute.provisioning_state != "Succeeded":
|
294
|
-
raise RuntimeError(
|
295
|
-
f"The provisioning state '{compute.provisioning_state}'"
|
296
|
-
f"of the compute cluster `{compute_name}` is not "
|
297
|
-
"successful. Please make sure that the compute cluster "
|
298
|
-
"is provisioned properly, before executing the "
|
299
|
-
"pipeline."
|
300
|
-
)
|
301
|
-
|
302
|
-
for parameter in [
|
303
|
-
"idle_time_before_scale_down",
|
304
|
-
"max_instances",
|
305
|
-
"min_instances",
|
306
|
-
"tier",
|
307
|
-
"location",
|
308
|
-
]:
|
309
|
-
# Check all possible configurations
|
310
|
-
self._check_settings_and_compute_configuration(
|
311
|
-
parameter=parameter, settings=settings, compute=compute
|
312
|
-
)
|
313
|
-
else:
|
314
|
-
raise RuntimeError(f"Unsupported compute type: {compute_type}")
|
315
|
-
return compute_name
|
316
|
-
|
317
|
-
# If the compute target does not exist create it
|
318
|
-
except ResourceNotFoundError:
|
319
|
-
logger.info(
|
320
|
-
"Can not find the compute target with name: "
|
321
|
-
f"'{compute_name}':"
|
322
|
-
)
|
323
|
-
|
324
|
-
if settings.mode == AzureMLComputeTypes.COMPUTE_INSTANCE:
|
325
|
-
logger.info(
|
326
|
-
"Creating a new compute instance. This might take a "
|
327
|
-
"few minutes."
|
328
|
-
)
|
329
|
-
|
330
|
-
from azure.ai.ml.entities import ComputeInstance
|
331
|
-
|
332
|
-
compute_instance = ComputeInstance(
|
333
|
-
name=compute_name,
|
334
|
-
size=settings.size,
|
335
|
-
idle_time_before_shutdown_minutes=settings.idle_time_before_shutdown_minutes,
|
336
|
-
)
|
337
|
-
client.begin_create_or_update(compute_instance).result()
|
338
|
-
return compute_name
|
339
|
-
|
340
|
-
elif settings.mode == AzureMLComputeTypes.COMPUTE_CLUSTER:
|
341
|
-
logger.info(
|
342
|
-
"Creating a new compute cluster. This might take a "
|
343
|
-
"few minutes."
|
344
|
-
)
|
345
|
-
|
346
|
-
from azure.ai.ml.entities import AmlCompute
|
347
|
-
|
348
|
-
compute_cluster = AmlCompute(
|
349
|
-
name=compute_name,
|
350
|
-
size=settings.size,
|
351
|
-
location=settings.location,
|
352
|
-
min_instances=settings.min_instances,
|
353
|
-
max_instances=settings.max_instances,
|
354
|
-
idle_time_before_scale_down=settings.idle_time_before_scaledown_down,
|
355
|
-
tier=settings.tier,
|
356
|
-
)
|
357
|
-
client.begin_create_or_update(compute_cluster).result()
|
358
|
-
return compute_name
|
359
|
-
|
360
|
-
return None
|
361
|
-
|
362
197
|
def prepare_or_run_pipeline(
|
363
198
|
self,
|
364
199
|
deployment: "PipelineDeploymentResponse",
|
@@ -429,7 +264,9 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
|
|
429
264
|
)
|
430
265
|
pipeline_args["name"] = run_name
|
431
266
|
|
432
|
-
if compute_target :=
|
267
|
+
if compute_target := create_or_get_compute(
|
268
|
+
ml_client, settings, default_compute_name=f"zenml_{self.id}"
|
269
|
+
):
|
433
270
|
pipeline_args["compute"] = compute_target
|
434
271
|
|
435
272
|
@pipeline(force_rerun=True, **pipeline_args) # type: ignore[call-overload, misc]
|
@@ -540,5 +377,54 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
|
|
540
377
|
)
|
541
378
|
|
542
379
|
else:
|
543
|
-
ml_client.jobs.create_or_update(pipeline_job)
|
380
|
+
job = ml_client.jobs.create_or_update(pipeline_job)
|
544
381
|
logger.info(f"Pipeline {run_name} has been started.")
|
382
|
+
assert job.services is not None
|
383
|
+
assert job.name is not None
|
384
|
+
|
385
|
+
logger.info(
|
386
|
+
f"Pipeline {run_name} is running. "
|
387
|
+
"You can view the pipeline in the AzureML portal at "
|
388
|
+
f"{job.services['Studio'].endpoint}"
|
389
|
+
)
|
390
|
+
|
391
|
+
if self.config.synchronous:
|
392
|
+
logger.info("Waiting for pipeline to finish...")
|
393
|
+
ml_client.jobs.stream(job.name)
|
394
|
+
|
395
|
+
def get_pipeline_run_metadata(
|
396
|
+
self, run_id: UUID
|
397
|
+
) -> Dict[str, "MetadataType"]:
|
398
|
+
"""Get general component-specific metadata for a pipeline run.
|
399
|
+
|
400
|
+
Args:
|
401
|
+
run_id: The ID of the pipeline run.
|
402
|
+
|
403
|
+
Returns:
|
404
|
+
A dictionary of metadata.
|
405
|
+
"""
|
406
|
+
try:
|
407
|
+
if connector := self.get_connector():
|
408
|
+
credentials = connector.connect()
|
409
|
+
else:
|
410
|
+
credentials = DefaultAzureCredential()
|
411
|
+
|
412
|
+
ml_client = MLClient(
|
413
|
+
credential=credentials,
|
414
|
+
subscription_id=self.config.subscription_id,
|
415
|
+
resource_group_name=self.config.resource_group,
|
416
|
+
workspace_name=self.config.workspace,
|
417
|
+
)
|
418
|
+
|
419
|
+
azureml_root_run_id = os.environ[ENV_ZENML_AZUREML_RUN_ID]
|
420
|
+
azureml_job = ml_client.jobs.get(azureml_root_run_id)
|
421
|
+
|
422
|
+
return {
|
423
|
+
METADATA_ORCHESTRATOR_URL: Uri(azureml_job.studio_url),
|
424
|
+
}
|
425
|
+
except Exception as e:
|
426
|
+
logger.warning(
|
427
|
+
f"Failed to fetch the Studio URL of the AzureML pipeline "
|
428
|
+
f"job: {e}"
|
429
|
+
)
|
430
|
+
return {}
|
@@ -69,6 +69,7 @@ class AzureMLEntrypointConfiguration(StepEntrypointConfiguration):
|
|
69
69
|
|
70
70
|
# Azure automatically changes the working directory, we have to set it
|
71
71
|
# back to /app before running the step.
|
72
|
+
os.makedirs("/app", exist_ok=True)
|
72
73
|
os.chdir("/app")
|
73
74
|
|
74
75
|
# Run the step
|
@@ -14,6 +14,7 @@
|
|
14
14
|
"""Azure Service Connector."""
|
15
15
|
|
16
16
|
import datetime
|
17
|
+
import logging
|
17
18
|
import re
|
18
19
|
import subprocess
|
19
20
|
from typing import Any, Dict, List, Optional, Tuple
|
@@ -58,6 +59,9 @@ from zenml.service_connectors.service_connector import (
|
|
58
59
|
from zenml.utils.enum_utils import StrEnum
|
59
60
|
from zenml.utils.secret_utils import PlainSerializedSecretStr
|
60
61
|
|
62
|
+
# Configure the logging level for azure.identity
|
63
|
+
logging.getLogger("azure.identity").setLevel(logging.WARNING)
|
64
|
+
|
61
65
|
logger = get_logger(__name__)
|
62
66
|
|
63
67
|
|
@@ -13,29 +13,16 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Implementation of the ZenML AzureML Step Operator."""
|
15
15
|
|
16
|
-
import itertools
|
17
|
-
import os
|
18
16
|
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type, cast
|
19
17
|
|
20
|
-
from
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
ScriptRunConfig,
|
25
|
-
Workspace,
|
26
|
-
)
|
27
|
-
from azureml.core.authentication import (
|
28
|
-
AbstractAuthentication,
|
29
|
-
ServicePrincipalAuthentication,
|
30
|
-
)
|
31
|
-
from azureml.core.conda_dependencies import CondaDependencies
|
18
|
+
from azure.ai.ml import MLClient, command
|
19
|
+
from azure.ai.ml.entities import Environment
|
20
|
+
from azure.core.credentials import TokenCredential
|
21
|
+
from azure.identity import ClientSecretCredential, DefaultAzureCredential
|
32
22
|
|
33
|
-
import
|
34
|
-
from zenml.
|
35
|
-
from zenml.
|
36
|
-
ENV_ZENML_CONFIG_PATH,
|
37
|
-
)
|
38
|
-
from zenml.environment import Environment as ZenMLEnvironment
|
23
|
+
from zenml.config.build_configuration import BuildConfiguration
|
24
|
+
from zenml.enums import StackComponentType
|
25
|
+
from zenml.integrations.azure.azureml_utils import create_or_get_compute
|
39
26
|
from zenml.integrations.azure.flavors.azureml_step_operator_flavor import (
|
40
27
|
AzureMLStepOperatorConfig,
|
41
28
|
AzureMLStepOperatorSettings,
|
@@ -43,20 +30,18 @@ from zenml.integrations.azure.flavors.azureml_step_operator_flavor import (
|
|
43
30
|
from zenml.logger import get_logger
|
44
31
|
from zenml.stack import Stack, StackValidator
|
45
32
|
from zenml.step_operators import BaseStepOperator
|
46
|
-
from zenml.utils import source_utils
|
47
|
-
from zenml.utils.pipeline_docker_image_builder import (
|
48
|
-
DOCKER_IMAGE_ZENML_CONFIG_DIR,
|
49
|
-
PipelineDockerImageBuilder,
|
50
|
-
)
|
51
33
|
|
52
34
|
if TYPE_CHECKING:
|
53
|
-
from zenml.config import DockerSettings
|
54
35
|
from zenml.config.base_settings import BaseSettings
|
55
36
|
from zenml.config.step_run_info import StepRunInfo
|
37
|
+
from zenml.models import PipelineDeploymentBase
|
56
38
|
|
57
39
|
logger = get_logger(__name__)
|
58
40
|
|
59
41
|
|
42
|
+
AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY = "azureml_step_operator"
|
43
|
+
|
44
|
+
|
60
45
|
class AzureMLStepOperator(BaseStepOperator):
|
61
46
|
"""Step operator to run a step on AzureML.
|
62
47
|
|
@@ -87,11 +72,11 @@ class AzureMLStepOperator(BaseStepOperator):
|
|
87
72
|
"""Validates the stack.
|
88
73
|
|
89
74
|
Returns:
|
90
|
-
A validator that checks that the stack contains a remote
|
91
|
-
store.
|
75
|
+
A validator that checks that the stack contains a remote container
|
76
|
+
registry and a remote artifact store.
|
92
77
|
"""
|
93
78
|
|
94
|
-
def
|
79
|
+
def _validate_remote_components(
|
95
80
|
stack: "Stack",
|
96
81
|
) -> Tuple[bool, str]:
|
97
82
|
if stack.artifact_store.config.is_local:
|
@@ -104,113 +89,75 @@ class AzureMLStepOperator(BaseStepOperator):
|
|
104
89
|
"step operator."
|
105
90
|
)
|
106
91
|
|
92
|
+
container_registry = stack.container_registry
|
93
|
+
assert container_registry is not None
|
94
|
+
|
95
|
+
if container_registry.config.is_local:
|
96
|
+
return False, (
|
97
|
+
"The AzureML step operator runs code remotely and "
|
98
|
+
"needs to push/pull Docker images, but the "
|
99
|
+
f"container registry `{container_registry.name}` of the "
|
100
|
+
"active stack is local. Please ensure that your stack "
|
101
|
+
"contains a remote container registry when using the "
|
102
|
+
"AzureML step operator."
|
103
|
+
)
|
104
|
+
|
107
105
|
return True, ""
|
108
106
|
|
109
107
|
return StackValidator(
|
110
|
-
|
108
|
+
required_components={
|
109
|
+
StackComponentType.CONTAINER_REGISTRY,
|
110
|
+
StackComponentType.IMAGE_BUILDER,
|
111
|
+
},
|
112
|
+
custom_validation_function=_validate_remote_components,
|
111
113
|
)
|
112
114
|
|
113
|
-
def
|
115
|
+
def _get_credentials(self) -> TokenCredential:
|
114
116
|
"""Returns the authentication object for the AzureML environment.
|
115
117
|
|
116
118
|
Returns:
|
117
119
|
The authentication object for the AzureML environment.
|
118
120
|
"""
|
119
|
-
|
121
|
+
# Authentication
|
122
|
+
if connector := self.get_connector():
|
123
|
+
credentials = connector.connect()
|
124
|
+
assert isinstance(credentials, TokenCredential)
|
125
|
+
return credentials
|
126
|
+
elif (
|
120
127
|
self.config.tenant_id
|
121
128
|
and self.config.service_principal_id
|
122
129
|
and self.config.service_principal_password
|
123
130
|
):
|
124
|
-
return
|
131
|
+
return ClientSecretCredential(
|
125
132
|
tenant_id=self.config.tenant_id,
|
126
|
-
|
127
|
-
|
133
|
+
client_id=self.config.service_principal_id,
|
134
|
+
client_secret=self.config.service_principal_password,
|
128
135
|
)
|
129
|
-
|
136
|
+
else:
|
137
|
+
return DefaultAzureCredential()
|
130
138
|
|
131
|
-
def
|
132
|
-
self,
|
133
|
-
|
134
|
-
|
135
|
-
run_name: str,
|
136
|
-
environment_variables: Dict[str, str],
|
137
|
-
environment_name: Optional[str] = None,
|
138
|
-
) -> Environment:
|
139
|
-
"""Prepares the environment in which Azure will run all jobs.
|
139
|
+
def get_docker_builds(
|
140
|
+
self, deployment: "PipelineDeploymentBase"
|
141
|
+
) -> List["BuildConfiguration"]:
|
142
|
+
"""Gets the Docker builds required for the component.
|
140
143
|
|
141
144
|
Args:
|
142
|
-
|
143
|
-
for a storage account, container registry among other
|
144
|
-
things.
|
145
|
-
docker_settings: The Docker settings for this step.
|
146
|
-
run_name: The name of the pipeline run that can be used
|
147
|
-
for naming environments and runs.
|
148
|
-
environment_variables: Environment variables to set in the
|
149
|
-
environment.
|
150
|
-
environment_name: Optional name of an existing environment to use.
|
145
|
+
deployment: The pipeline deployment for which to get the builds.
|
151
146
|
|
152
147
|
Returns:
|
153
|
-
The
|
148
|
+
The required Docker builds.
|
154
149
|
"""
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
itertools.chain.from_iterable(
|
163
|
-
r[1].split("\n") for r in requirements_files
|
164
|
-
)
|
165
|
-
)
|
166
|
-
requirements.append(f"zenml=={zenml.__version__}")
|
167
|
-
logger.info(
|
168
|
-
"Using requirements for AzureML step operator environment: %s",
|
169
|
-
requirements,
|
170
|
-
)
|
171
|
-
if environment_name:
|
172
|
-
environment = Environment.get(
|
173
|
-
workspace=workspace, name=environment_name
|
174
|
-
)
|
175
|
-
if not environment.python.conda_dependencies:
|
176
|
-
environment.python.conda_dependencies = (
|
177
|
-
CondaDependencies.create(
|
178
|
-
python_version=ZenMLEnvironment.python_version()
|
179
|
-
)
|
150
|
+
builds = []
|
151
|
+
for step_name, step in deployment.step_configurations.items():
|
152
|
+
if step.config.step_operator == self.name:
|
153
|
+
build = BuildConfiguration(
|
154
|
+
key=AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY,
|
155
|
+
settings=step.config.docker_settings,
|
156
|
+
step_name=step_name,
|
180
157
|
)
|
158
|
+
builds.append(build)
|
181
159
|
|
182
|
-
|
183
|
-
environment.python.conda_dependencies.add_pip_package(
|
184
|
-
requirement
|
185
|
-
)
|
186
|
-
else:
|
187
|
-
environment = Environment(name=f"zenml-{run_name}")
|
188
|
-
environment.python.conda_dependencies = CondaDependencies.create(
|
189
|
-
pip_packages=requirements,
|
190
|
-
python_version=ZenMLEnvironment.python_version(),
|
191
|
-
)
|
192
|
-
|
193
|
-
if docker_settings.parent_image:
|
194
|
-
# replace the default azure base image
|
195
|
-
environment.docker.base_image = docker_settings.parent_image
|
196
|
-
|
197
|
-
# set credentials to access azure storage
|
198
|
-
for key in [
|
199
|
-
"AZURE_STORAGE_ACCOUNT_KEY",
|
200
|
-
"AZURE_STORAGE_ACCOUNT_NAME",
|
201
|
-
"AZURE_STORAGE_CONNECTION_STRING",
|
202
|
-
"AZURE_STORAGE_SAS_TOKEN",
|
203
|
-
]:
|
204
|
-
value = os.getenv(key)
|
205
|
-
if value:
|
206
|
-
environment_variables[key] = value
|
207
|
-
|
208
|
-
environment_variables[ENV_ZENML_CONFIG_PATH] = (
|
209
|
-
f"./{DOCKER_IMAGE_ZENML_CONFIG_DIR}"
|
210
|
-
)
|
211
|
-
environment_variables.update(docker_settings.environment)
|
212
|
-
environment.environment_variables = environment_variables
|
213
|
-
return environment
|
160
|
+
return builds
|
214
161
|
|
215
162
|
def launch(
|
216
163
|
self,
|
@@ -226,79 +173,33 @@ class AzureMLStepOperator(BaseStepOperator):
|
|
226
173
|
environment: Environment variables to set in the step operator
|
227
174
|
environment.
|
228
175
|
"""
|
229
|
-
if not info.config.resource_settings.empty:
|
230
|
-
logger.warning(
|
231
|
-
"Specifying custom step resources is not supported for "
|
232
|
-
"the AzureML step operator. If you want to run this step "
|
233
|
-
"operator on specific resources, you can do so by creating an "
|
234
|
-
"Azure compute target (https://docs.microsoft.com/en-us/azure/machine-learning/concept-compute-target) "
|
235
|
-
"with a specific machine type and then updating this step "
|
236
|
-
"operator: `zenml step-operator update %s "
|
237
|
-
"--compute_target_name=<COMPUTE_TARGET_NAME>`",
|
238
|
-
self.name,
|
239
|
-
)
|
240
|
-
|
241
|
-
unused_docker_fields = [
|
242
|
-
"dockerfile",
|
243
|
-
"build_context_root",
|
244
|
-
"build_options",
|
245
|
-
"skip_build",
|
246
|
-
"target_repository",
|
247
|
-
"dockerignore",
|
248
|
-
"copy_files",
|
249
|
-
"copy_global_config",
|
250
|
-
"apt_packages",
|
251
|
-
"user",
|
252
|
-
"source_files",
|
253
|
-
"allow_including_files_in_images",
|
254
|
-
"allow_download_from_code_repository",
|
255
|
-
"allow_download_from_artifact_store",
|
256
|
-
]
|
257
|
-
docker_settings = info.config.docker_settings
|
258
|
-
ignored_docker_fields = docker_settings.model_fields_set.intersection(
|
259
|
-
unused_docker_fields
|
260
|
-
)
|
261
|
-
|
262
|
-
if ignored_docker_fields:
|
263
|
-
logger.warning(
|
264
|
-
"The AzureML step operator currently does not support all "
|
265
|
-
"options defined in your Docker settings. Ignoring all "
|
266
|
-
"values set for the attributes: %s",
|
267
|
-
ignored_docker_fields,
|
268
|
-
)
|
269
|
-
|
270
176
|
settings = cast(AzureMLStepOperatorSettings, self.get_settings(info))
|
177
|
+
image_name = info.get_image(key=AZUREML_STEP_OPERATOR_DOCKER_IMAGE_KEY)
|
271
178
|
|
272
|
-
|
179
|
+
# Client creation
|
180
|
+
ml_client = MLClient(
|
181
|
+
credential=self._get_credentials(),
|
273
182
|
subscription_id=self.config.subscription_id,
|
274
|
-
|
275
|
-
|
276
|
-
auth=self._get_authentication(),
|
183
|
+
resource_group_name=self.config.resource_group,
|
184
|
+
workspace_name=self.config.workspace_name,
|
277
185
|
)
|
278
186
|
|
279
|
-
|
187
|
+
env = Environment(name=f"zenml-{info.run_name}", image=image_name)
|
280
188
|
|
281
|
-
|
282
|
-
|
283
|
-
docker_settings=docker_settings,
|
284
|
-
run_name=info.run_name,
|
285
|
-
environment_variables=environment,
|
286
|
-
environment_name=settings.environment_name,
|
287
|
-
)
|
288
|
-
compute_target = ComputeTarget(
|
289
|
-
workspace=workspace, name=self.config.compute_target_name
|
189
|
+
compute_target = create_or_get_compute(
|
190
|
+
ml_client, settings, default_compute_name=f"zenml_{self.id}"
|
290
191
|
)
|
291
192
|
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
193
|
+
command_job = command(
|
194
|
+
name=info.run_name,
|
195
|
+
command=" ".join(entrypoint_command),
|
196
|
+
environment=env,
|
197
|
+
environment_variables=environment,
|
198
|
+
compute=compute_target,
|
199
|
+
experiment_name=info.pipeline.name,
|
297
200
|
)
|
298
201
|
|
299
|
-
|
300
|
-
run = experiment.submit(config=run_config)
|
202
|
+
job = ml_client.jobs.create_or_update(command_job)
|
301
203
|
|
302
|
-
|
303
|
-
info.
|
304
|
-
run.wait_for_completion(show_output=True)
|
204
|
+
logger.info(f"AzureML job created with id: {job.id}")
|
205
|
+
ml_client.jobs.stream(info.run_name)
|
zenml/integrations/constants.py
CHANGED
@@ -44,7 +44,9 @@ LIGHTGBM = "lightgbm"
|
|
44
44
|
MLFLOW = "mlflow"
|
45
45
|
NEPTUNE = "neptune"
|
46
46
|
NEURAL_PROPHET = "neural_prophet"
|
47
|
+
NUMPY = "numpy"
|
47
48
|
OPEN_AI = "openai"
|
49
|
+
PANDAS = "pandas"
|
48
50
|
PIGEON = "pigeon"
|
49
51
|
PILLOW = "pillow"
|
50
52
|
PLOTLY = "plotly"
|
@@ -73,3 +75,4 @@ WANDB = "wandb"
|
|
73
75
|
VERTEX = "vertex"
|
74
76
|
XGBOOST = "xgboost"
|
75
77
|
VAULT = "vault"
|
78
|
+
LIGHTNING = "lightning"
|