zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,139 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""AzureML definitions."""
|
15
|
+
|
16
|
+
from typing import Optional
|
17
|
+
|
18
|
+
from pydantic import model_validator
|
19
|
+
|
20
|
+
from zenml.config.base_settings import BaseSettings
|
21
|
+
from zenml.logger import get_logger
|
22
|
+
from zenml.utils.enum_utils import StrEnum
|
23
|
+
|
24
|
+
logger = get_logger(__name__)
|
25
|
+
|
26
|
+
|
27
|
+
class AzureMLComputeTypes(StrEnum):
|
28
|
+
"""Enum for different types of compute on AzureML."""
|
29
|
+
|
30
|
+
SERVERLESS = "serverless"
|
31
|
+
COMPUTE_INSTANCE = "compute-instance"
|
32
|
+
COMPUTE_CLUSTER = "compute-cluster"
|
33
|
+
|
34
|
+
|
35
|
+
class AzureMLComputeSettings(BaseSettings):
|
36
|
+
"""Settings for the AzureML compute.
|
37
|
+
|
38
|
+
These settings adjust the compute resources that will be used by the
|
39
|
+
pipeline execution.
|
40
|
+
|
41
|
+
There are three possible use cases for this implementation:
|
42
|
+
|
43
|
+
1. Serverless compute (default behavior):
|
44
|
+
- The `mode` is set to `serverless` (default behavior).
|
45
|
+
- All the other parameters become irrelevant and will throw a
|
46
|
+
warning if set.
|
47
|
+
|
48
|
+
2. Compute instance:
|
49
|
+
- The `mode` is set to `compute-instance`.
|
50
|
+
- In this case, users have to provide a `compute-name`.
|
51
|
+
- If a compute instance exists with this name, this instance
|
52
|
+
will be used and all the other parameters become irrelevant
|
53
|
+
and will throw a warning if set.
|
54
|
+
- If a compute instance does not already exist, ZenML will
|
55
|
+
create it. You can use the parameters `compute_size` and
|
56
|
+
`idle_type_before_shutdown_minutes` for this operation.
|
57
|
+
|
58
|
+
3. Compute cluster:
|
59
|
+
- The `mode` is set to `compute-cluster`.
|
60
|
+
- In this case, users have to provide a `compute-name`.
|
61
|
+
- If a compute cluster exists with this name, this instance
|
62
|
+
will be used and all the other parameters become irrelevant
|
63
|
+
and will throw a warning if set.
|
64
|
+
- If a compute cluster does not already exist, ZenML will
|
65
|
+
create it. You can set the additional parameters for this
|
66
|
+
operation.
|
67
|
+
"""
|
68
|
+
|
69
|
+
# Mode for compute
|
70
|
+
mode: AzureMLComputeTypes = AzureMLComputeTypes.SERVERLESS
|
71
|
+
|
72
|
+
# Common Configuration for Compute Instances and Clusters
|
73
|
+
compute_name: Optional[str] = None
|
74
|
+
size: Optional[str] = None
|
75
|
+
|
76
|
+
# Additional configuration for a Compute Instance
|
77
|
+
idle_time_before_shutdown_minutes: Optional[int] = None
|
78
|
+
|
79
|
+
# Additional configuration for a Compute Cluster
|
80
|
+
idle_time_before_scaledown_down: Optional[int] = None
|
81
|
+
location: Optional[str] = None
|
82
|
+
min_instances: Optional[int] = None
|
83
|
+
max_instances: Optional[int] = None
|
84
|
+
tier: Optional[str] = None
|
85
|
+
|
86
|
+
@model_validator(mode="after")
|
87
|
+
def azureml_settings_validator(self) -> "AzureMLComputeSettings":
|
88
|
+
"""Checks whether the right configuration is set based on mode.
|
89
|
+
|
90
|
+
Returns:
|
91
|
+
the instance itself.
|
92
|
+
|
93
|
+
Raises:
|
94
|
+
AssertionError: if a name is not provided when working with
|
95
|
+
instances and clusters.
|
96
|
+
"""
|
97
|
+
viable_configuration_fields = {
|
98
|
+
AzureMLComputeTypes.SERVERLESS: {"mode"},
|
99
|
+
AzureMLComputeTypes.COMPUTE_INSTANCE: {
|
100
|
+
"mode",
|
101
|
+
"compute_name",
|
102
|
+
"size",
|
103
|
+
"idle_time_before_shutdown_minutes",
|
104
|
+
},
|
105
|
+
AzureMLComputeTypes.COMPUTE_CLUSTER: {
|
106
|
+
"mode",
|
107
|
+
"compute_name",
|
108
|
+
"size",
|
109
|
+
"idle_time_before_scaledown_down",
|
110
|
+
"location",
|
111
|
+
"min_instances",
|
112
|
+
"max_instances",
|
113
|
+
"tier",
|
114
|
+
},
|
115
|
+
}
|
116
|
+
viable_fields = viable_configuration_fields[self.mode]
|
117
|
+
|
118
|
+
for field in self.model_fields_set:
|
119
|
+
if (
|
120
|
+
field not in viable_fields
|
121
|
+
and field in AzureMLComputeSettings.model_fields
|
122
|
+
):
|
123
|
+
logger.warning(
|
124
|
+
f"In the {self.__class__.__name__} settings, the mode of "
|
125
|
+
f"operation is set to {self.mode}. In this mode, you can "
|
126
|
+
f"not configure the parameter '{field}'. This "
|
127
|
+
"configuration will be ignored."
|
128
|
+
)
|
129
|
+
|
130
|
+
if (
|
131
|
+
self.mode == AzureMLComputeTypes.COMPUTE_INSTANCE
|
132
|
+
or self.mode == AzureMLComputeTypes.COMPUTE_CLUSTER
|
133
|
+
):
|
134
|
+
assert self.compute_name is not None, (
|
135
|
+
"When you are working with compute instances and clusters, "
|
136
|
+
"please define a name for the compute target."
|
137
|
+
)
|
138
|
+
|
139
|
+
return self
|
@@ -15,20 +15,19 @@
|
|
15
15
|
|
16
16
|
from typing import TYPE_CHECKING, Optional, Type
|
17
17
|
|
18
|
-
from pydantic import Field
|
18
|
+
from pydantic import Field
|
19
19
|
|
20
|
-
from zenml.config.base_settings import BaseSettings
|
21
20
|
from zenml.integrations.azure import (
|
22
21
|
AZURE_RESOURCE_TYPE,
|
23
22
|
AZUREML_ORCHESTRATOR_FLAVOR,
|
24
23
|
)
|
24
|
+
from zenml.integrations.azure.flavors.azureml import AzureMLComputeSettings
|
25
25
|
from zenml.logger import get_logger
|
26
26
|
from zenml.models import ServiceConnectorRequirements
|
27
27
|
from zenml.orchestrators.base_orchestrator import (
|
28
28
|
BaseOrchestratorConfig,
|
29
29
|
BaseOrchestratorFlavor,
|
30
30
|
)
|
31
|
-
from zenml.utils.enum_utils import StrEnum
|
32
31
|
|
33
32
|
if TYPE_CHECKING:
|
34
33
|
from zenml.integrations.azure.orchestrators import AzureMLOrchestrator
|
@@ -36,118 +35,8 @@ if TYPE_CHECKING:
|
|
36
35
|
logger = get_logger(__name__)
|
37
36
|
|
38
37
|
|
39
|
-
class
|
40
|
-
"""
|
41
|
-
|
42
|
-
SERVERLESS = "serverless"
|
43
|
-
COMPUTE_INSTANCE = "compute-instance"
|
44
|
-
COMPUTE_CLUSTER = "compute-cluster"
|
45
|
-
|
46
|
-
|
47
|
-
class AzureMLOrchestratorSettings(BaseSettings):
|
48
|
-
"""Settings for the AzureML orchestrator.
|
49
|
-
|
50
|
-
These settings adjust the compute resources that will be used by the
|
51
|
-
pipeline execution.
|
52
|
-
|
53
|
-
There are three possible use cases for this implementation:
|
54
|
-
|
55
|
-
1. Serverless compute (default behaviour):
|
56
|
-
- The `mode` is set to `serverless` (default behaviour).
|
57
|
-
- All the other parameters become irrelevant and will throw a
|
58
|
-
warning if set.
|
59
|
-
|
60
|
-
2. Compute instance:
|
61
|
-
- The `mode` is set to `compute-instance`.
|
62
|
-
- In this case, users have to provide a `compute-name`.
|
63
|
-
- If a compute instance exists with this name, this instance
|
64
|
-
will be used and all the other parameters become irrelevant
|
65
|
-
and will throw a warning if set.
|
66
|
-
- If a compute instance does not already exist, ZenML will
|
67
|
-
create it. You can use the parameters `compute_size` and
|
68
|
-
`idle_type_before_shutdown_minutes` for this operation.
|
69
|
-
|
70
|
-
3. Compute cluster:
|
71
|
-
- The `mode` is set to `compute-cluster`.
|
72
|
-
- In this case, users have to provide a `compute-name`.
|
73
|
-
- If a compute cluster exists with this name, this instance
|
74
|
-
will be used and all the other parameters become irrelevant
|
75
|
-
and will throw a warning if set.
|
76
|
-
- If a compute cluster does not already exist, ZenML will
|
77
|
-
create it. You can all the additional parameters for this
|
78
|
-
operation.
|
79
|
-
"""
|
80
|
-
|
81
|
-
# Mode for compute
|
82
|
-
mode: AzureMLComputeTypes = AzureMLComputeTypes.SERVERLESS
|
83
|
-
|
84
|
-
# Common Configuration for Compute Instances and Clusters
|
85
|
-
compute_name: Optional[str] = None
|
86
|
-
size: Optional[str] = None
|
87
|
-
|
88
|
-
# Additional configuration for a Compute Instance
|
89
|
-
idle_time_before_shutdown_minutes: Optional[int] = None
|
90
|
-
|
91
|
-
# Additional configuration for a Compute Cluster
|
92
|
-
idle_time_before_scaledown_down: Optional[int] = None
|
93
|
-
location: Optional[str] = None
|
94
|
-
min_instances: Optional[int] = None
|
95
|
-
max_instances: Optional[int] = None
|
96
|
-
tier: Optional[str] = None
|
97
|
-
|
98
|
-
@model_validator(mode="after")
|
99
|
-
def azureml_settings_validator(self) -> "AzureMLOrchestratorSettings":
|
100
|
-
"""Checks whether the right configuration is set based on mode.
|
101
|
-
|
102
|
-
Returns:
|
103
|
-
the instance itself.
|
104
|
-
|
105
|
-
Raises:
|
106
|
-
AssertionError: if a name is not provided when working with
|
107
|
-
instances and clusters.
|
108
|
-
"""
|
109
|
-
excluded_fields = {"subscription_id", "resource_group", "workspace"}
|
110
|
-
|
111
|
-
viable_configuration_fields = {
|
112
|
-
AzureMLComputeTypes.SERVERLESS: {"mode"},
|
113
|
-
AzureMLComputeTypes.COMPUTE_INSTANCE: {
|
114
|
-
"mode",
|
115
|
-
"compute_name",
|
116
|
-
"size",
|
117
|
-
"idle_time_before_shutdown_minutes",
|
118
|
-
},
|
119
|
-
AzureMLComputeTypes.COMPUTE_CLUSTER: {
|
120
|
-
"mode",
|
121
|
-
"compute_name",
|
122
|
-
"size",
|
123
|
-
"idle_time_before_scaledown_down",
|
124
|
-
"location",
|
125
|
-
"min_instances",
|
126
|
-
"max_instances",
|
127
|
-
"tier",
|
128
|
-
},
|
129
|
-
}
|
130
|
-
viable_fields = viable_configuration_fields[self.mode]
|
131
|
-
|
132
|
-
for field in self.model_fields_set:
|
133
|
-
if field not in viable_fields and field not in excluded_fields:
|
134
|
-
logger.warning(
|
135
|
-
"In the AzureML Orchestrator Settings, the mode of "
|
136
|
-
f"operation is set to {self.mode}. In this mode, you can "
|
137
|
-
f"not configure the parameter '{field}'. This "
|
138
|
-
"configuration will be ignored."
|
139
|
-
)
|
140
|
-
|
141
|
-
if (
|
142
|
-
self.mode == AzureMLComputeTypes.COMPUTE_INSTANCE
|
143
|
-
or self.mode == AzureMLComputeTypes.COMPUTE_CLUSTER
|
144
|
-
):
|
145
|
-
assert self.compute_name is not None, (
|
146
|
-
"When you are working with compute instances and clusters, "
|
147
|
-
"please define a name for the compute target."
|
148
|
-
)
|
149
|
-
|
150
|
-
return self
|
38
|
+
class AzureMLOrchestratorSettings(AzureMLComputeSettings):
|
39
|
+
"""Settings for the AzureML orchestrator."""
|
151
40
|
|
152
41
|
|
153
42
|
class AzureMLOrchestratorConfig(
|
@@ -164,6 +53,10 @@ class AzureMLOrchestratorConfig(
|
|
164
53
|
workspace: str = Field(
|
165
54
|
description="Name of the workspace that AzureML is running on."
|
166
55
|
)
|
56
|
+
synchronous: bool = Field(
|
57
|
+
default=True,
|
58
|
+
description="Whether the orchestrator runs synchronously or not.",
|
59
|
+
)
|
167
60
|
|
168
61
|
@property
|
169
62
|
def is_remote(self) -> bool:
|
@@ -180,12 +73,21 @@ class AzureMLOrchestratorConfig(
|
|
180
73
|
|
181
74
|
@property
|
182
75
|
def is_synchronous(self) -> bool:
|
183
|
-
"""Whether the orchestrator runs
|
76
|
+
"""Whether the orchestrator runs synchronously or not.
|
184
77
|
|
185
78
|
Returns:
|
186
|
-
Whether the orchestrator runs
|
79
|
+
Whether the orchestrator runs synchronously or not.
|
187
80
|
"""
|
188
|
-
return
|
81
|
+
return self.synchronous
|
82
|
+
|
83
|
+
@property
|
84
|
+
def is_schedulable(self) -> bool:
|
85
|
+
"""Whether the orchestrator is schedulable or not.
|
86
|
+
|
87
|
+
Returns:
|
88
|
+
Whether the orchestrator is schedulable or not.
|
89
|
+
"""
|
90
|
+
return True
|
189
91
|
|
190
92
|
|
191
93
|
class AzureMLOrchestratorFlavor(BaseOrchestratorFlavor):
|
@@ -13,29 +13,65 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""AzureML step operator flavor."""
|
15
15
|
|
16
|
-
from typing import TYPE_CHECKING, Optional, Type
|
16
|
+
from typing import TYPE_CHECKING, Any, Dict, Optional, Type
|
17
17
|
|
18
|
-
from
|
19
|
-
|
18
|
+
from pydantic import Field, model_validator
|
19
|
+
|
20
|
+
from zenml.integrations.azure import (
|
21
|
+
AZURE_RESOURCE_TYPE,
|
22
|
+
AZUREML_STEP_OPERATOR_FLAVOR,
|
23
|
+
)
|
24
|
+
from zenml.integrations.azure.flavors.azureml import (
|
25
|
+
AzureMLComputeSettings,
|
26
|
+
AzureMLComputeTypes,
|
27
|
+
)
|
28
|
+
from zenml.models import ServiceConnectorRequirements
|
20
29
|
from zenml.step_operators.base_step_operator import (
|
21
30
|
BaseStepOperatorConfig,
|
22
31
|
BaseStepOperatorFlavor,
|
23
32
|
)
|
33
|
+
from zenml.utils.pydantic_utils import before_validator_handler
|
24
34
|
from zenml.utils.secret_utils import SecretField
|
25
35
|
|
26
36
|
if TYPE_CHECKING:
|
27
37
|
from zenml.integrations.azure.step_operators import AzureMLStepOperator
|
28
38
|
|
29
39
|
|
30
|
-
class AzureMLStepOperatorSettings(
|
40
|
+
class AzureMLStepOperatorSettings(AzureMLComputeSettings):
|
31
41
|
"""Settings for the AzureML step operator.
|
32
42
|
|
33
43
|
Attributes:
|
34
|
-
|
35
|
-
|
44
|
+
compute_target_name: The name of the configured ComputeTarget.
|
45
|
+
Deprecated in favor of `compute_name`.
|
36
46
|
"""
|
37
47
|
|
38
|
-
|
48
|
+
compute_target_name: Optional[str] = Field(
|
49
|
+
default=None,
|
50
|
+
description="Name of the configured ComputeTarget. Deprecated in favor "
|
51
|
+
"of `compute_name`.",
|
52
|
+
)
|
53
|
+
|
54
|
+
@model_validator(mode="before")
|
55
|
+
@classmethod
|
56
|
+
@before_validator_handler
|
57
|
+
def _migrate_compute_name(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
58
|
+
"""Backward compatibility for compute_target_name.
|
59
|
+
|
60
|
+
Args:
|
61
|
+
data: The model data.
|
62
|
+
|
63
|
+
Returns:
|
64
|
+
The migrated data.
|
65
|
+
"""
|
66
|
+
if (
|
67
|
+
"compute_target_name" in data
|
68
|
+
and "compute_name" not in data
|
69
|
+
and "mode" not in data
|
70
|
+
):
|
71
|
+
data["compute_name"] = data.pop("compute_target_name")
|
72
|
+
data["mode"] = AzureMLComputeTypes.COMPUTE_INSTANCE
|
73
|
+
|
74
|
+
return data
|
39
75
|
|
40
76
|
|
41
77
|
class AzureMLStepOperatorConfig(
|
@@ -48,19 +84,21 @@ class AzureMLStepOperatorConfig(
|
|
48
84
|
resource_group: The resource group to which the AzureML workspace
|
49
85
|
is deployed.
|
50
86
|
workspace_name: The name of the AzureML Workspace.
|
51
|
-
compute_target_name: The name of the configured ComputeTarget.
|
52
|
-
An instance of it has to be created on the portal if it doesn't
|
53
|
-
exist already.
|
54
87
|
tenant_id: The Azure Tenant ID.
|
55
88
|
service_principal_id: The ID for the service principal that is created
|
56
89
|
to allow apps to access secure resources.
|
57
90
|
service_principal_password: Password for the service principal.
|
58
91
|
"""
|
59
92
|
|
60
|
-
subscription_id: str
|
61
|
-
|
62
|
-
|
63
|
-
|
93
|
+
subscription_id: str = Field(
|
94
|
+
description="Subscription ID that AzureML is running on."
|
95
|
+
)
|
96
|
+
resource_group: str = Field(
|
97
|
+
description="Name of the resource group that AzureML is running on.",
|
98
|
+
)
|
99
|
+
workspace_name: str = Field(
|
100
|
+
description="Name of the workspace that AzureML is running on."
|
101
|
+
)
|
64
102
|
|
65
103
|
# Service principal authentication
|
66
104
|
# https://docs.microsoft.com/en-us/azure/machine-learning/how-to-setup-authentication#configure-a-service-principal
|
@@ -94,6 +132,21 @@ class AzureMLStepOperatorFlavor(BaseStepOperatorFlavor):
|
|
94
132
|
"""
|
95
133
|
return AZUREML_STEP_OPERATOR_FLAVOR
|
96
134
|
|
135
|
+
@property
|
136
|
+
def service_connector_requirements(
|
137
|
+
self,
|
138
|
+
) -> Optional[ServiceConnectorRequirements]:
|
139
|
+
"""Service connector resource requirements for service connectors.
|
140
|
+
|
141
|
+
Specifies resource requirements that are used to filter the available
|
142
|
+
service connector types that are compatible with this flavor.
|
143
|
+
|
144
|
+
Returns:
|
145
|
+
Requirements for compatible service connectors, if a service
|
146
|
+
connector is required for this flavor.
|
147
|
+
"""
|
148
|
+
return ServiceConnectorRequirements(resource_type=AZURE_RESOURCE_TYPE)
|
149
|
+
|
97
150
|
@property
|
98
151
|
def docs_url(self) -> Optional[str]:
|
99
152
|
"""A url to point at docs explaining this flavor.
|