zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
zenml/config/server_config.py
CHANGED
@@ -381,6 +381,36 @@ class ServerConfiguration(BaseModel):
|
|
381
381
|
# Revert to the default value if the header is enabled
|
382
382
|
del data[k]
|
383
383
|
|
384
|
+
# Handle merging of user-defined secure_headers_csp value with the default value
|
385
|
+
if "secure_headers_csp" in data:
|
386
|
+
user_defined_csp = data["secure_headers_csp"]
|
387
|
+
if isinstance(user_defined_csp, str):
|
388
|
+
# Parse the user-defined CSP string into a dictionary
|
389
|
+
user_defined_csp_dict = {}
|
390
|
+
for directive in user_defined_csp.split(";"):
|
391
|
+
directive = directive.strip()
|
392
|
+
if directive:
|
393
|
+
key, value = directive.split(" ", 1)
|
394
|
+
user_defined_csp_dict[key] = value.strip("'\"")
|
395
|
+
|
396
|
+
# Merge the user-defined CSP dictionary with the default CSP dictionary
|
397
|
+
default_csp_dict = {}
|
398
|
+
for directive in DEFAULT_ZENML_SERVER_SECURE_HEADERS_CSP.split(
|
399
|
+
";"
|
400
|
+
):
|
401
|
+
directive = directive.strip()
|
402
|
+
if directive:
|
403
|
+
key, value = directive.split(" ", 1)
|
404
|
+
default_csp_dict[key] = value.strip("'\"")
|
405
|
+
|
406
|
+
merged_csp_dict = {**default_csp_dict, **user_defined_csp_dict}
|
407
|
+
|
408
|
+
# Convert the merged CSP dictionary back to a string
|
409
|
+
merged_csp_str = "; ".join(
|
410
|
+
f"{key} {value}" for key, value in merged_csp_dict.items()
|
411
|
+
)
|
412
|
+
data["secure_headers_csp"] = merged_csp_str
|
413
|
+
|
384
414
|
return data
|
385
415
|
|
386
416
|
@property
|
zenml/config/source.py
CHANGED
@@ -234,20 +234,16 @@ class NotebookSource(Source):
|
|
234
234
|
"""Source representing an object defined in a notebook.
|
235
235
|
|
236
236
|
Attributes:
|
237
|
-
code_path: Path where the notebook cell code for this source is
|
238
|
-
uploaded.
|
239
237
|
replacement_module: Name of the module from which this source should
|
240
238
|
be loaded in case the code is not running in a notebook.
|
239
|
+
artifact_store_id: ID of the artifact store in which the replacement
|
240
|
+
module code is stored.
|
241
241
|
"""
|
242
242
|
|
243
|
-
code_path: Optional[str] = None
|
244
243
|
replacement_module: Optional[str] = None
|
244
|
+
artifact_store_id: Optional[UUID] = None
|
245
245
|
type: SourceType = SourceType.NOTEBOOK
|
246
246
|
|
247
|
-
# Private attribute that is used to store the code but should not be
|
248
|
-
# serialized
|
249
|
-
_cell_code: Optional[str] = None
|
250
|
-
|
251
247
|
@field_validator("type")
|
252
248
|
@classmethod
|
253
249
|
def _validate_type(cls, value: SourceType) -> SourceType:
|
zenml/constants.py
CHANGED
@@ -162,7 +162,6 @@ ENV_ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING = (
|
|
162
162
|
)
|
163
163
|
ENV_ZENML_DISABLE_WORKSPACE_WARNINGS = "ZENML_DISABLE_WORKSPACE_WARNINGS"
|
164
164
|
ENV_ZENML_SKIP_IMAGE_BUILDER_DEFAULT = "ZENML_SKIP_IMAGE_BUILDER_DEFAULT"
|
165
|
-
ENV_ZENML_REQUIRES_CODE_DOWNLOAD = "ZENML_REQUIRES_CODE_DOWNLOAD"
|
166
165
|
ENV_ZENML_SERVER = "ZENML_SERVER"
|
167
166
|
ENV_ZENML_LOCAL_SERVER = "ZENML_LOCAL_SERVER"
|
168
167
|
ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
|
@@ -173,6 +172,7 @@ ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
|
|
173
172
|
)
|
174
173
|
ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
|
175
174
|
ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
|
175
|
+
ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
|
176
176
|
|
177
177
|
# ZenML Server environment variables
|
178
178
|
ENV_ZENML_SERVER_PREFIX = "ZENML_SERVER_"
|
@@ -185,6 +185,9 @@ ENV_ZENML_SERVER_USE_LEGACY_DASHBOARD = (
|
|
185
185
|
f"{ENV_ZENML_SERVER_PREFIX}USE_LEGACY_DASHBOARD"
|
186
186
|
)
|
187
187
|
ENV_ZENML_SERVER_AUTO_ACTIVATE = f"{ENV_ZENML_SERVER_PREFIX}AUTO_ACTIVATE"
|
188
|
+
ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK = (
|
189
|
+
"ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK"
|
190
|
+
)
|
188
191
|
|
189
192
|
# Logging variables
|
190
193
|
IS_DEBUG_ENV: bool = handle_bool_env_var(ENV_ZENML_DEBUG, default=False)
|
@@ -348,7 +351,6 @@ EMAIL_ANALYTICS = "/email-opt-in"
|
|
348
351
|
EVENT_FLAVORS = "/event-flavors"
|
349
352
|
EVENT_SOURCES = "/event-sources"
|
350
353
|
FLAVORS = "/flavors"
|
351
|
-
FULL_STACK = "/full-stack"
|
352
354
|
GET_OR_CREATE = "/get-or-create"
|
353
355
|
GRAPH = "/graph"
|
354
356
|
HEALTH = "/health"
|
@@ -486,7 +488,7 @@ MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION = (
|
|
486
488
|
)
|
487
489
|
|
488
490
|
|
489
|
-
FINISHED_ONBOARDING_SURVEY_KEY = "
|
491
|
+
FINISHED_ONBOARDING_SURVEY_KEY = "finished_onboarding_survey"
|
490
492
|
|
491
493
|
# Name validation
|
492
494
|
BANNED_NAME_CHARACTERS = "\t\n\r\v\f"
|
@@ -17,15 +17,11 @@ import argparse
|
|
17
17
|
import os
|
18
18
|
import sys
|
19
19
|
from abc import ABC, abstractmethod
|
20
|
-
from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Set
|
20
|
+
from typing import TYPE_CHECKING, Any, Dict, List, NoReturn, Optional, Set
|
21
21
|
from uuid import UUID
|
22
22
|
|
23
23
|
from zenml.client import Client
|
24
24
|
from zenml.code_repositories import BaseCodeRepository
|
25
|
-
from zenml.constants import (
|
26
|
-
ENV_ZENML_REQUIRES_CODE_DOWNLOAD,
|
27
|
-
handle_bool_env_var,
|
28
|
-
)
|
29
25
|
from zenml.logger import get_logger
|
30
26
|
from zenml.utils import (
|
31
27
|
code_repository_utils,
|
@@ -194,22 +190,28 @@ class BaseEntrypointConfiguration(ABC):
|
|
194
190
|
return Client().zen_store.get_deployment(deployment_id=deployment_id)
|
195
191
|
|
196
192
|
def download_code_if_necessary(
|
197
|
-
self,
|
193
|
+
self,
|
194
|
+
deployment: "PipelineDeploymentResponse",
|
195
|
+
step_name: Optional[str] = None,
|
198
196
|
) -> None:
|
199
197
|
"""Downloads user code if necessary.
|
200
198
|
|
201
199
|
Args:
|
202
200
|
deployment: The deployment for which to download the code.
|
201
|
+
step_name: Name of the step to be run. This will be used to
|
202
|
+
determine whether code download is necessary. If not given,
|
203
|
+
the DockerSettings of the pipeline will be used to make that
|
204
|
+
decision instead.
|
203
205
|
|
204
206
|
Raises:
|
205
207
|
RuntimeError: If the current environment requires code download
|
206
208
|
but the deployment does not have a reference to any code.
|
207
209
|
"""
|
208
|
-
|
209
|
-
|
210
|
+
should_download_code = self._should_download_code(
|
211
|
+
deployment=deployment, step_name=step_name
|
210
212
|
)
|
211
213
|
|
212
|
-
if not
|
214
|
+
if not should_download_code:
|
213
215
|
return
|
214
216
|
|
215
217
|
if code_reference := deployment.code_reference:
|
@@ -217,7 +219,7 @@ class BaseEntrypointConfiguration(ABC):
|
|
217
219
|
code_reference=code_reference
|
218
220
|
)
|
219
221
|
elif code_path := deployment.code_path:
|
220
|
-
|
222
|
+
code_utils.download_code_from_artifact_store(code_path=code_path)
|
221
223
|
else:
|
222
224
|
raise RuntimeError(
|
223
225
|
"Code download required but no code reference or path provided."
|
@@ -259,30 +261,42 @@ class BaseEntrypointConfiguration(ABC):
|
|
259
261
|
sys.path.insert(0, download_dir)
|
260
262
|
os.chdir(download_dir)
|
261
263
|
|
262
|
-
def
|
263
|
-
|
264
|
+
def _should_download_code(
|
265
|
+
self,
|
266
|
+
deployment: "PipelineDeploymentResponse",
|
267
|
+
step_name: Optional[str] = None,
|
268
|
+
) -> bool:
|
269
|
+
"""Checks whether code should be downloaded.
|
264
270
|
|
265
271
|
Args:
|
266
|
-
|
272
|
+
deployment: The deployment to check.
|
273
|
+
step_name: Name of the step to be run. This will be used to
|
274
|
+
determine whether code download is necessary. If not given,
|
275
|
+
the DockerSettings of the pipeline will be used to make that
|
276
|
+
decision instead.
|
277
|
+
|
278
|
+
Returns:
|
279
|
+
Whether code should be downloaded.
|
267
280
|
"""
|
268
|
-
|
269
|
-
|
281
|
+
docker_settings = (
|
282
|
+
deployment.step_configurations[step_name].config.docker_settings
|
283
|
+
if step_name
|
284
|
+
else deployment.pipeline_configuration.docker_settings
|
270
285
|
)
|
271
286
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
os.makedirs(extract_dir)
|
287
|
+
if (
|
288
|
+
deployment.code_reference
|
289
|
+
and docker_settings.allow_download_from_code_repository
|
290
|
+
):
|
291
|
+
return True
|
278
292
|
|
279
|
-
|
280
|
-
code_path
|
281
|
-
|
293
|
+
if (
|
294
|
+
deployment.code_path
|
295
|
+
and docker_settings.allow_download_from_artifact_store
|
296
|
+
):
|
297
|
+
return True
|
282
298
|
|
283
|
-
|
284
|
-
sys.path.insert(0, extract_dir)
|
285
|
-
os.chdir(extract_dir)
|
299
|
+
return False
|
286
300
|
|
287
301
|
@abstractmethod
|
288
302
|
def run(self) -> None:
|
@@ -151,9 +151,12 @@ class StepEntrypointConfiguration(BaseEntrypointConfiguration):
|
|
151
151
|
# and stack component flavors are registered.
|
152
152
|
integration_registry.activate_integrations()
|
153
153
|
|
154
|
-
self.download_code_if_necessary(deployment=deployment)
|
155
|
-
|
156
154
|
step_name = self.entrypoint_args[STEP_NAME_OPTION]
|
155
|
+
|
156
|
+
self.download_code_if_necessary(
|
157
|
+
deployment=deployment, step_name=step_name
|
158
|
+
)
|
159
|
+
|
157
160
|
pipeline_name = deployment.pipeline_configuration.name
|
158
161
|
|
159
162
|
step = deployment.step_configurations[step_name]
|
zenml/enums.py
CHANGED
@@ -303,6 +303,8 @@ class EnvironmentType(StrEnum):
|
|
303
303
|
PAPERSPACE = "paperspace"
|
304
304
|
WSL = "wsl"
|
305
305
|
LIGHTNING_AI_STUDIO = "lightning_ai_studio"
|
306
|
+
GITHUB_CODESPACES = "github_codespaces"
|
307
|
+
VSCODE_REMOTE_CONTAINER = "vscode_remote_container"
|
306
308
|
|
307
309
|
|
308
310
|
class ModelStages(StrEnum):
|
zenml/environment.py
CHANGED
@@ -69,6 +69,10 @@ def get_environment() -> str:
|
|
69
69
|
return EnvironmentType.BITBUCKET_CI
|
70
70
|
elif Environment.in_ci():
|
71
71
|
return EnvironmentType.GENERIC_CI
|
72
|
+
elif Environment.in_github_codespaces():
|
73
|
+
return EnvironmentType.GITHUB_CODESPACES
|
74
|
+
elif Environment.in_vscode_remote_container():
|
75
|
+
return EnvironmentType.VSCODE_REMOTE_CONTAINER
|
72
76
|
elif Environment.in_lightning_ai_studio():
|
73
77
|
return EnvironmentType.LIGHTNING_AI_STUDIO
|
74
78
|
elif Environment.in_docker():
|
@@ -271,6 +275,33 @@ class Environment(metaclass=SingletonMetaClass):
|
|
271
275
|
return True
|
272
276
|
return False
|
273
277
|
|
278
|
+
@staticmethod
|
279
|
+
def in_github_codespaces() -> bool:
|
280
|
+
"""If the current Python process is running in GitHub Codespaces.
|
281
|
+
|
282
|
+
Returns:
|
283
|
+
`True` if the current Python process is running in GitHub Codespaces,
|
284
|
+
`False` otherwise.
|
285
|
+
"""
|
286
|
+
return (
|
287
|
+
"CODESPACES" in os.environ
|
288
|
+
or "GITHUB_CODESPACE_TOKEN" in os.environ
|
289
|
+
or "GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN" in os.environ
|
290
|
+
)
|
291
|
+
|
292
|
+
@staticmethod
|
293
|
+
def in_vscode_remote_container() -> bool:
|
294
|
+
"""If the current Python process is running in a VS Code Remote Container.
|
295
|
+
|
296
|
+
Returns:
|
297
|
+
`True` if the current Python process is running in a VS Code Remote Container,
|
298
|
+
`False` otherwise.
|
299
|
+
"""
|
300
|
+
return (
|
301
|
+
"REMOTE_CONTAINERS" in os.environ
|
302
|
+
or "VSCODE_REMOTE_CONTAINERS_SESSION" in os.environ
|
303
|
+
)
|
304
|
+
|
274
305
|
@staticmethod
|
275
306
|
def in_paperspace_gradient() -> bool:
|
276
307
|
"""If the current Python process is running in Paperspace Gradient.
|
@@ -14,9 +14,7 @@
|
|
14
14
|
"""The base class for feature stores."""
|
15
15
|
|
16
16
|
from abc import ABC, abstractmethod
|
17
|
-
from typing import Any, Dict, List, Type,
|
18
|
-
|
19
|
-
import pandas as pd
|
17
|
+
from typing import Any, Dict, List, Type, cast
|
20
18
|
|
21
19
|
from zenml.enums import StackComponentType
|
22
20
|
from zenml.stack import Flavor, StackComponent
|
@@ -42,10 +40,10 @@ class BaseFeatureStore(StackComponent, ABC):
|
|
42
40
|
@abstractmethod
|
43
41
|
def get_historical_features(
|
44
42
|
self,
|
45
|
-
entity_df:
|
43
|
+
entity_df: Any,
|
46
44
|
features: List[str],
|
47
45
|
full_feature_names: bool = False,
|
48
|
-
) ->
|
46
|
+
) -> Any:
|
49
47
|
"""Returns the historical features for training or batch scoring.
|
50
48
|
|
51
49
|
Args:
|
@@ -54,7 +52,7 @@ class BaseFeatureStore(StackComponent, ABC):
|
|
54
52
|
full_feature_names: Whether to return the full feature names.
|
55
53
|
|
56
54
|
Returns:
|
57
|
-
The historical features
|
55
|
+
The historical features.
|
58
56
|
"""
|
59
57
|
|
60
58
|
@abstractmethod
|
zenml/integrations/__init__.py
CHANGED
@@ -36,6 +36,7 @@ from zenml.integrations.gitlab import GitLabIntegration # noqa
|
|
36
36
|
from zenml.integrations.great_expectations import ( # noqa
|
37
37
|
GreatExpectationsIntegration,
|
38
38
|
)
|
39
|
+
from zenml.integrations.lightning import LightningIntegration # noqa
|
39
40
|
from zenml.integrations.huggingface import HuggingfaceIntegration # noqa
|
40
41
|
from zenml.integrations.hyperai import HyperAIIntegration # noqa
|
41
42
|
from zenml.integrations.kaniko import KanikoIntegration # noqa
|
@@ -48,7 +49,9 @@ from zenml.integrations.lightgbm import LightGBMIntegration # noqa
|
|
48
49
|
from zenml.integrations.mlflow import MlflowIntegration # noqa
|
49
50
|
from zenml.integrations.neptune import NeptuneIntegration # noqa
|
50
51
|
from zenml.integrations.neural_prophet import NeuralProphetIntegration # noqa
|
52
|
+
from zenml.integrations.numpy import NumpyIntegration # noqa
|
51
53
|
from zenml.integrations.openai import OpenAIIntegration # noqa
|
54
|
+
from zenml.integrations.pandas import PandasIntegration # noqa
|
52
55
|
from zenml.integrations.pigeon import PigeonIntegration # noqa
|
53
56
|
from zenml.integrations.pillow import PillowIntegration # noqa
|
54
57
|
from zenml.integrations.polars import PolarsIntegration # noqa
|
@@ -119,6 +119,15 @@ class AirflowOrchestratorConfig(
|
|
119
119
|
|
120
120
|
local: bool = True
|
121
121
|
|
122
|
+
@property
|
123
|
+
def is_schedulable(self) -> bool:
|
124
|
+
"""Whether the orchestrator is schedulable or not.
|
125
|
+
|
126
|
+
Returns:
|
127
|
+
Whether the orchestrator is schedulable or not.
|
128
|
+
"""
|
129
|
+
return True
|
130
|
+
|
122
131
|
|
123
132
|
class AirflowOrchestratorFlavor(BaseOrchestratorFlavor):
|
124
133
|
"""Flavor for the Airflow orchestrator."""
|
@@ -45,8 +45,8 @@ class AWSIntegration(Integration):
|
|
45
45
|
]
|
46
46
|
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
|
47
47
|
|
48
|
-
@
|
49
|
-
def activate() -> None:
|
48
|
+
@classmethod
|
49
|
+
def activate(cls) -> None:
|
50
50
|
"""Activate the AWS integration."""
|
51
51
|
from zenml.integrations.aws import service_connectors # noqa
|
52
52
|
|
@@ -372,7 +372,8 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
372
372
|
# mainly for testing purposes, we wait for the pipeline to finish
|
373
373
|
if self.config.synchronous:
|
374
374
|
logger.info(
|
375
|
-
"Executing synchronously. Waiting for pipeline to finish..."
|
375
|
+
"Executing synchronously. Waiting for pipeline to finish... \n"
|
376
|
+
"At this point you can `Ctrl-C` out without cancelling the execution."
|
376
377
|
)
|
377
378
|
try:
|
378
379
|
pipeline_execution.wait(
|
@@ -52,8 +52,8 @@ class AzureIntegration(Integration):
|
|
52
52
|
]
|
53
53
|
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
|
54
54
|
|
55
|
-
@
|
56
|
-
def activate() -> None:
|
55
|
+
@classmethod
|
56
|
+
def activate(cls) -> None:
|
57
57
|
"""Activate the Azure integration."""
|
58
58
|
from zenml.integrations.azure import service_connectors # noqa
|
59
59
|
|
@@ -0,0 +1,201 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""AzureML definitions."""
|
15
|
+
|
16
|
+
from typing import Optional
|
17
|
+
|
18
|
+
from azure.ai.ml import MLClient
|
19
|
+
from azure.ai.ml.entities import Compute
|
20
|
+
from azure.core.exceptions import (
|
21
|
+
ResourceNotFoundError,
|
22
|
+
)
|
23
|
+
|
24
|
+
from zenml.integrations.azure.flavors.azureml import (
|
25
|
+
AzureMLComputeSettings,
|
26
|
+
AzureMLComputeTypes,
|
27
|
+
)
|
28
|
+
from zenml.logger import get_logger
|
29
|
+
|
30
|
+
logger = get_logger(__name__)
|
31
|
+
|
32
|
+
|
33
|
+
def check_settings_and_compute_configuration(
|
34
|
+
parameter: str,
|
35
|
+
settings: AzureMLComputeSettings,
|
36
|
+
compute: Compute,
|
37
|
+
) -> None:
|
38
|
+
"""Utility function comparing a parameter between settings and compute.
|
39
|
+
|
40
|
+
Args:
|
41
|
+
parameter: the name of the parameter.
|
42
|
+
settings: The AzureML orchestrator settings.
|
43
|
+
compute: The compute instance or cluster from AzureML.
|
44
|
+
"""
|
45
|
+
# Check the compute size
|
46
|
+
compute_value = getattr(compute, parameter)
|
47
|
+
settings_value = getattr(settings, parameter)
|
48
|
+
|
49
|
+
if settings_value is not None and settings_value != compute_value:
|
50
|
+
logger.warning(
|
51
|
+
f"The '{parameter}' defined in the settings '{settings_value}' "
|
52
|
+
"does not match the actual parameter of the instance: "
|
53
|
+
f"'{compute_value}'. Will ignore this setting for now."
|
54
|
+
)
|
55
|
+
|
56
|
+
|
57
|
+
def create_or_get_compute(
|
58
|
+
client: MLClient,
|
59
|
+
settings: AzureMLComputeSettings,
|
60
|
+
default_compute_name: str,
|
61
|
+
) -> Optional[str]:
|
62
|
+
"""Creates or fetches the compute target if defined in the settings.
|
63
|
+
|
64
|
+
Args:
|
65
|
+
client: the AzureML client.
|
66
|
+
settings: the settings for the orchestrator.
|
67
|
+
default_compute_name: the default name for the compute target, if one
|
68
|
+
is not provided in the settings.
|
69
|
+
|
70
|
+
Returns:
|
71
|
+
None, if the orchestrator is using serverless compute or
|
72
|
+
str, the name of the compute target (instance or cluster).
|
73
|
+
|
74
|
+
Raises:
|
75
|
+
RuntimeError: if the fetched compute target is unsupported or the
|
76
|
+
mode defined in the setting does not match the type of the
|
77
|
+
compute target.
|
78
|
+
"""
|
79
|
+
# If the mode is serverless, we can not fetch anything anyhow
|
80
|
+
if settings.mode == AzureMLComputeTypes.SERVERLESS:
|
81
|
+
return None
|
82
|
+
|
83
|
+
# If a name is not provided, generate one based on the orchestrator id
|
84
|
+
compute_name = settings.compute_name or default_compute_name
|
85
|
+
|
86
|
+
# Try to fetch the compute target
|
87
|
+
try:
|
88
|
+
compute = client.compute.get(compute_name)
|
89
|
+
|
90
|
+
logger.info(f"Using existing compute target: '{compute_name}'.")
|
91
|
+
|
92
|
+
# Check if compute size matches with the settings
|
93
|
+
check_settings_and_compute_configuration(
|
94
|
+
parameter="size", settings=settings, compute=compute
|
95
|
+
)
|
96
|
+
|
97
|
+
compute_type = compute.type
|
98
|
+
|
99
|
+
# Check the type and matches the settings
|
100
|
+
if compute_type == "computeinstance": # Compute Instance
|
101
|
+
if settings.mode != AzureMLComputeTypes.COMPUTE_INSTANCE:
|
102
|
+
raise RuntimeError(
|
103
|
+
"The mode of operation for the compute target defined"
|
104
|
+
f"in the settings '{settings.mode}' does not match "
|
105
|
+
f"the type of the compute target: `{compute_name}` "
|
106
|
+
"which is a 'compute-instance'. Please make sure that "
|
107
|
+
"the settings are adjusted properly."
|
108
|
+
)
|
109
|
+
|
110
|
+
if compute.state != "Running":
|
111
|
+
raise RuntimeError(
|
112
|
+
f"The compute instance `{compute_name}` is not in a "
|
113
|
+
"running state at the moment. Please make sure that "
|
114
|
+
"the compute target is running, before executing the "
|
115
|
+
"pipeline."
|
116
|
+
)
|
117
|
+
|
118
|
+
# Idle time before shutdown
|
119
|
+
check_settings_and_compute_configuration(
|
120
|
+
parameter="idle_time_before_shutdown_minutes",
|
121
|
+
settings=settings,
|
122
|
+
compute=compute,
|
123
|
+
)
|
124
|
+
|
125
|
+
elif compute_type == "amIcompute": # Compute Cluster
|
126
|
+
if settings.mode != AzureMLComputeTypes.COMPUTE_CLUSTER:
|
127
|
+
raise RuntimeError(
|
128
|
+
"The mode of operation for the compute target defined "
|
129
|
+
f"in the settings '{settings.mode}' does not match "
|
130
|
+
f"the type of the compute target: `{compute_name}` "
|
131
|
+
"which is a 'compute-cluster'. Please make sure that "
|
132
|
+
"the settings are adjusted properly."
|
133
|
+
)
|
134
|
+
|
135
|
+
if compute.provisioning_state != "Succeeded":
|
136
|
+
raise RuntimeError(
|
137
|
+
f"The provisioning state '{compute.provisioning_state}'"
|
138
|
+
f"of the compute cluster `{compute_name}` is not "
|
139
|
+
"successful. Please make sure that the compute cluster "
|
140
|
+
"is provisioned properly, before executing the "
|
141
|
+
"pipeline."
|
142
|
+
)
|
143
|
+
|
144
|
+
for parameter in [
|
145
|
+
"idle_time_before_scale_down",
|
146
|
+
"max_instances",
|
147
|
+
"min_instances",
|
148
|
+
"tier",
|
149
|
+
"location",
|
150
|
+
]:
|
151
|
+
# Check all possible configurations
|
152
|
+
check_settings_and_compute_configuration(
|
153
|
+
parameter=parameter, settings=settings, compute=compute
|
154
|
+
)
|
155
|
+
else:
|
156
|
+
raise RuntimeError(f"Unsupported compute type: {compute_type}")
|
157
|
+
return compute_name
|
158
|
+
|
159
|
+
# If the compute target does not exist create it
|
160
|
+
except ResourceNotFoundError:
|
161
|
+
logger.info(
|
162
|
+
"Can not find the compute target with name: " f"'{compute_name}':"
|
163
|
+
)
|
164
|
+
|
165
|
+
if settings.mode == AzureMLComputeTypes.COMPUTE_INSTANCE:
|
166
|
+
logger.info(
|
167
|
+
"Creating a new compute instance. This might take a "
|
168
|
+
"few minutes."
|
169
|
+
)
|
170
|
+
|
171
|
+
from azure.ai.ml.entities import ComputeInstance
|
172
|
+
|
173
|
+
compute_instance = ComputeInstance(
|
174
|
+
name=compute_name,
|
175
|
+
size=settings.size,
|
176
|
+
idle_time_before_shutdown_minutes=settings.idle_time_before_shutdown_minutes,
|
177
|
+
)
|
178
|
+
client.begin_create_or_update(compute_instance).result()
|
179
|
+
return compute_name
|
180
|
+
|
181
|
+
elif settings.mode == AzureMLComputeTypes.COMPUTE_CLUSTER:
|
182
|
+
logger.info(
|
183
|
+
"Creating a new compute cluster. This might take a "
|
184
|
+
"few minutes."
|
185
|
+
)
|
186
|
+
|
187
|
+
from azure.ai.ml.entities import AmlCompute
|
188
|
+
|
189
|
+
compute_cluster = AmlCompute(
|
190
|
+
name=compute_name,
|
191
|
+
size=settings.size,
|
192
|
+
location=settings.location,
|
193
|
+
min_instances=settings.min_instances,
|
194
|
+
max_instances=settings.max_instances,
|
195
|
+
idle_time_before_scale_down=settings.idle_time_before_scaledown_down,
|
196
|
+
tier=settings.tier,
|
197
|
+
)
|
198
|
+
client.begin_create_or_update(compute_cluster).result()
|
199
|
+
return compute_name
|
200
|
+
|
201
|
+
return None
|