zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,307 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Entrypoint of the Lightning master/orchestrator STUDIO."""
|
15
|
+
|
16
|
+
import argparse
|
17
|
+
import os
|
18
|
+
from typing import Dict, cast
|
19
|
+
|
20
|
+
from lightning_sdk import Machine, Studio
|
21
|
+
|
22
|
+
from zenml.client import Client
|
23
|
+
from zenml.entrypoints.step_entrypoint_configuration import (
|
24
|
+
StepEntrypointConfiguration,
|
25
|
+
)
|
26
|
+
from zenml.enums import ExecutionStatus
|
27
|
+
from zenml.integrations.lightning.flavors.lightning_orchestrator_flavor import (
|
28
|
+
LightningOrchestratorSettings,
|
29
|
+
)
|
30
|
+
from zenml.integrations.lightning.orchestrators.lightning_orchestrator import (
|
31
|
+
ENV_ZENML_LIGHTNING_ORCHESTRATOR_RUN_ID,
|
32
|
+
LightningOrchestrator,
|
33
|
+
)
|
34
|
+
from zenml.integrations.lightning.orchestrators.utils import (
|
35
|
+
gather_requirements,
|
36
|
+
sanitize_studio_name,
|
37
|
+
)
|
38
|
+
from zenml.logger import get_logger
|
39
|
+
from zenml.orchestrators.dag_runner import ThreadedDagRunner
|
40
|
+
|
41
|
+
logger = get_logger(__name__)
|
42
|
+
|
43
|
+
|
44
|
+
LIGHTNING_ZENML_DEFAULT_CUSTOM_REPOSITORY_PATH = "."
|
45
|
+
|
46
|
+
|
47
|
+
def parse_args() -> argparse.Namespace:
|
48
|
+
"""Parse entrypoint arguments.
|
49
|
+
|
50
|
+
Returns:
|
51
|
+
Parsed args.
|
52
|
+
"""
|
53
|
+
parser = argparse.ArgumentParser()
|
54
|
+
parser.add_argument("--run_name", type=str, required=True)
|
55
|
+
parser.add_argument("--deployment_id", type=str, required=True)
|
56
|
+
return parser.parse_args()
|
57
|
+
|
58
|
+
|
59
|
+
def main() -> None:
|
60
|
+
"""Entrypoint of the Lightning master/orchestrator STUDIO.
|
61
|
+
|
62
|
+
This is the entrypoint of the Lightning master/orchestrator STUDIO. It is
|
63
|
+
responsible for provisioning the STUDIO and running the pipeline steps in
|
64
|
+
separate STUDIO.
|
65
|
+
|
66
|
+
Raises:
|
67
|
+
TypeError: If the active stack's orchestrator is not an instance of
|
68
|
+
LightningOrchestrator.
|
69
|
+
ValueError: If the active stack's container registry is None.
|
70
|
+
"""
|
71
|
+
# Log to the container's stdout so it can be streamed by the client.
|
72
|
+
logger.info("Lightning orchestrator STUDIO started.")
|
73
|
+
|
74
|
+
# Parse / extract args.
|
75
|
+
args = parse_args()
|
76
|
+
|
77
|
+
orchestrator_run_id = os.environ.get(
|
78
|
+
ENV_ZENML_LIGHTNING_ORCHESTRATOR_RUN_ID
|
79
|
+
)
|
80
|
+
if not orchestrator_run_id:
|
81
|
+
raise ValueError(
|
82
|
+
f"Environment variable '{ENV_ZENML_LIGHTNING_ORCHESTRATOR_RUN_ID}' is not set."
|
83
|
+
)
|
84
|
+
|
85
|
+
logger.info(f"Orchestrator run id: {orchestrator_run_id}")
|
86
|
+
|
87
|
+
deployment = Client().get_deployment(args.deployment_id)
|
88
|
+
filename = f"{args.run_name}.tar.gz"
|
89
|
+
|
90
|
+
pipeline_dag = {
|
91
|
+
step_name: step.spec.upstream_steps
|
92
|
+
for step_name, step in deployment.step_configurations.items()
|
93
|
+
}
|
94
|
+
entrypoint_command = StepEntrypointConfiguration.get_entrypoint_command()
|
95
|
+
|
96
|
+
active_stack = Client().active_stack
|
97
|
+
|
98
|
+
orchestrator = active_stack.orchestrator
|
99
|
+
if not isinstance(orchestrator, LightningOrchestrator):
|
100
|
+
raise TypeError(
|
101
|
+
"The active stack's orchestrator is not an instance of LightningOrchestrator."
|
102
|
+
)
|
103
|
+
|
104
|
+
# Set up credentials
|
105
|
+
orchestrator._set_lightning_env_vars(deployment)
|
106
|
+
|
107
|
+
pipeline_settings = cast(
|
108
|
+
LightningOrchestratorSettings, orchestrator.get_settings(deployment)
|
109
|
+
)
|
110
|
+
|
111
|
+
# Gather the requirements
|
112
|
+
pipeline_docker_settings = (
|
113
|
+
deployment.pipeline_configuration.docker_settings
|
114
|
+
)
|
115
|
+
pipeline_requirements = gather_requirements(pipeline_docker_settings)
|
116
|
+
pipeline_requirements_to_string = " ".join(
|
117
|
+
f'"{req}"' for req in pipeline_requirements
|
118
|
+
)
|
119
|
+
|
120
|
+
unique_resource_configs: Dict[str, str] = {}
|
121
|
+
main_studio_name = sanitize_studio_name(
|
122
|
+
f"zenml_{orchestrator_run_id}_pipeline"
|
123
|
+
)
|
124
|
+
for step_name, step in deployment.step_configurations.items():
|
125
|
+
step_settings = cast(
|
126
|
+
LightningOrchestratorSettings,
|
127
|
+
orchestrator.get_settings(step),
|
128
|
+
)
|
129
|
+
unique_resource_configs[step_name] = main_studio_name
|
130
|
+
if pipeline_settings.machine_type != step_settings.machine_type:
|
131
|
+
unique_resource_configs[step_name] = (
|
132
|
+
f"zenml-{orchestrator_run_id}_{step_name}"
|
133
|
+
)
|
134
|
+
|
135
|
+
logger.info(f"Creating main studio: {main_studio_name}")
|
136
|
+
main_studio = Studio(name=main_studio_name)
|
137
|
+
if pipeline_settings.machine_type:
|
138
|
+
main_studio.start(Machine(pipeline_settings.machine_type))
|
139
|
+
else:
|
140
|
+
main_studio.start()
|
141
|
+
|
142
|
+
logger.info("Main studio started.")
|
143
|
+
logger.info("Uploading code to main studio the code path: %s", filename)
|
144
|
+
main_studio.upload_file(
|
145
|
+
"/teamspace/studios/this_studio/.lightning_studio/.studiorc",
|
146
|
+
remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
|
147
|
+
)
|
148
|
+
output = main_studio.run(
|
149
|
+
f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
150
|
+
)
|
151
|
+
logger.info(output)
|
152
|
+
main_studio.upload_file(
|
153
|
+
f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
154
|
+
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
155
|
+
)
|
156
|
+
logger.info("Extracting code... ")
|
157
|
+
output = main_studio.run(
|
158
|
+
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
159
|
+
)
|
160
|
+
logger.info(f"Code extraction output: {output}")
|
161
|
+
logger.info("Installing requirements... ")
|
162
|
+
|
163
|
+
output = main_studio.run("pip install uv")
|
164
|
+
logger.info(output)
|
165
|
+
output = main_studio.run(
|
166
|
+
f"uv pip install {pipeline_requirements_to_string}"
|
167
|
+
)
|
168
|
+
logger.info(output)
|
169
|
+
output = main_studio.run(
|
170
|
+
"pip uninstall zenml -y && pip install git+https://github.com/zenml-io/zenml.git@feature/lightening-studio-orchestrator"
|
171
|
+
)
|
172
|
+
logger.info(output)
|
173
|
+
|
174
|
+
for command in pipeline_settings.custom_commands or []:
|
175
|
+
output = main_studio.run(
|
176
|
+
f"cd /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]} && {command}"
|
177
|
+
)
|
178
|
+
logger.info(f"Custom command output: {output}")
|
179
|
+
|
180
|
+
run = Client().list_pipeline_runs(
|
181
|
+
sort_by="asc:created",
|
182
|
+
size=1,
|
183
|
+
deployment_id=args.deployment_id,
|
184
|
+
status=ExecutionStatus.INITIALIZING,
|
185
|
+
)[0]
|
186
|
+
|
187
|
+
logger.info("Fetching pipeline run: %s", run.id)
|
188
|
+
|
189
|
+
def run_step_on_lightning_studio(step_name: str) -> None:
|
190
|
+
"""Run a pipeline step in a separate Lightning STUDIO.
|
191
|
+
|
192
|
+
Args:
|
193
|
+
step_name: Name of the step.
|
194
|
+
|
195
|
+
Raises:
|
196
|
+
Exception: If an error occurs while running the step on the STUDIO.
|
197
|
+
"""
|
198
|
+
step_args = StepEntrypointConfiguration.get_entrypoint_arguments(
|
199
|
+
step_name=step_name,
|
200
|
+
deployment_id=args.deployment_id,
|
201
|
+
)
|
202
|
+
|
203
|
+
entrypoint = entrypoint_command + step_args
|
204
|
+
entrypoint_string = " ".join(entrypoint)
|
205
|
+
run_command = f"{entrypoint_string}"
|
206
|
+
|
207
|
+
step = deployment.step_configurations[step_name]
|
208
|
+
if unique_resource_configs[step_name] != main_studio_name:
|
209
|
+
logger.info(
|
210
|
+
f"Creating separate studio for step: {unique_resource_configs[step_name]}"
|
211
|
+
)
|
212
|
+
# Get step settings
|
213
|
+
step_settings = cast(
|
214
|
+
LightningOrchestratorSettings,
|
215
|
+
orchestrator.get_settings(step),
|
216
|
+
)
|
217
|
+
# Gather the requirements
|
218
|
+
step_docker_settings = step.config.docker_settings
|
219
|
+
step_requirements = gather_requirements(step_docker_settings)
|
220
|
+
step_requirements_to_string = " ".join(
|
221
|
+
f'"{req}"' for req in step_requirements
|
222
|
+
)
|
223
|
+
run_command = f"{entrypoint_string}"
|
224
|
+
|
225
|
+
logger.info(
|
226
|
+
f"Creating separate studio for step: {unique_resource_configs[step_name]}"
|
227
|
+
)
|
228
|
+
studio = Studio(name=unique_resource_configs[step_name])
|
229
|
+
try:
|
230
|
+
studio.start(Machine(step_settings.machine_type))
|
231
|
+
output = studio.run(
|
232
|
+
f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
233
|
+
)
|
234
|
+
logger.info(output)
|
235
|
+
studio.upload_file(
|
236
|
+
f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
237
|
+
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
238
|
+
)
|
239
|
+
output = studio.run(
|
240
|
+
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
241
|
+
)
|
242
|
+
logger.info(output)
|
243
|
+
studio.upload_file(
|
244
|
+
"/teamspace/studios/this_studio/.lightning_studio/.studiorc",
|
245
|
+
remote_path="/teamspace/studios/this_studio/.lightning_studio/.studiorc",
|
246
|
+
)
|
247
|
+
output = studio.run("pip install uv")
|
248
|
+
logger.info(output)
|
249
|
+
output = studio.run(
|
250
|
+
f"uv pip install {step_requirements_to_string}"
|
251
|
+
)
|
252
|
+
logger.info(output)
|
253
|
+
output = studio.run(
|
254
|
+
"pip uninstall zenml -y && pip install git+https://github.com/zenml-io/zenml.git@feature/lightening-studio-orchestrator"
|
255
|
+
)
|
256
|
+
logger.info(output)
|
257
|
+
for command in step_settings.custom_commands or []:
|
258
|
+
output = studio.run(
|
259
|
+
f"cd /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]} && {command}"
|
260
|
+
)
|
261
|
+
logger.info(f"Custom command output: {output}")
|
262
|
+
output = studio.run(
|
263
|
+
f"cd /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]} && {run_command}"
|
264
|
+
)
|
265
|
+
logger.info(output)
|
266
|
+
except Exception as e:
|
267
|
+
logger.error(
|
268
|
+
f"Error running step {step_name} on studio {unique_resource_configs[step_name]}: {e}"
|
269
|
+
)
|
270
|
+
raise e
|
271
|
+
finally:
|
272
|
+
studio.delete()
|
273
|
+
studio.delete()
|
274
|
+
else:
|
275
|
+
output = main_studio.run(
|
276
|
+
f"cd /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]} && {run_command}"
|
277
|
+
)
|
278
|
+
logger.info(output)
|
279
|
+
|
280
|
+
# Pop the resource configuration for this step
|
281
|
+
unique_resource_configs.pop(step_name)
|
282
|
+
|
283
|
+
if main_studio_name in unique_resource_configs.values():
|
284
|
+
# If there are more steps using this configuration, skip deprovisioning the cluster
|
285
|
+
logger.info(
|
286
|
+
f"Resource configuration for studio '{main_studio_name}' "
|
287
|
+
"is used by subsequent steps. Skipping the deprovisioning of "
|
288
|
+
"the studio."
|
289
|
+
)
|
290
|
+
else:
|
291
|
+
# If there are no more steps using this configuration, down the cluster
|
292
|
+
logger.info(
|
293
|
+
f"Resource configuration for cluster '{main_studio_name}' "
|
294
|
+
"is not used by subsequent steps. deprovisioning the cluster."
|
295
|
+
)
|
296
|
+
main_studio.delete()
|
297
|
+
logger.info(f"Running step `{step_name}` on a Studio is completed.")
|
298
|
+
|
299
|
+
ThreadedDagRunner(
|
300
|
+
dag=pipeline_dag, run_fn=run_step_on_lightning_studio
|
301
|
+
).run()
|
302
|
+
|
303
|
+
logger.info("Orchestration STUDIO provisioned.")
|
304
|
+
|
305
|
+
|
306
|
+
if __name__ == "__main__":
|
307
|
+
main()
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Entrypoint configuration for the Lightning master/orchestrator VM."""
|
15
|
+
|
16
|
+
from typing import TYPE_CHECKING, List, Set
|
17
|
+
|
18
|
+
if TYPE_CHECKING:
|
19
|
+
from uuid import UUID
|
20
|
+
|
21
|
+
RUN_NAME_OPTION = "run_name"
|
22
|
+
DEPLOYMENT_ID_OPTION = "deployment_id"
|
23
|
+
|
24
|
+
|
25
|
+
class LightningOrchestratorEntrypointConfiguration:
|
26
|
+
"""Entrypoint configuration for the Lightning master/orchestrator VM."""
|
27
|
+
|
28
|
+
@classmethod
|
29
|
+
def get_entrypoint_options(cls) -> Set[str]:
|
30
|
+
"""Gets all the options required for running this entrypoint.
|
31
|
+
|
32
|
+
Returns:
|
33
|
+
Entrypoint options.
|
34
|
+
"""
|
35
|
+
options = {
|
36
|
+
RUN_NAME_OPTION,
|
37
|
+
DEPLOYMENT_ID_OPTION,
|
38
|
+
}
|
39
|
+
return options
|
40
|
+
|
41
|
+
@classmethod
|
42
|
+
def get_entrypoint_command(cls) -> List[str]:
|
43
|
+
"""Returns a command that runs the entrypoint module.
|
44
|
+
|
45
|
+
Returns:
|
46
|
+
Entrypoint command.
|
47
|
+
"""
|
48
|
+
command = [
|
49
|
+
"python",
|
50
|
+
"-m",
|
51
|
+
"zenml.integrations.lightning.orchestrators.lightning_orchestrator_entrypoint",
|
52
|
+
]
|
53
|
+
return command
|
54
|
+
|
55
|
+
@classmethod
|
56
|
+
def get_entrypoint_arguments(
|
57
|
+
cls,
|
58
|
+
run_name: str,
|
59
|
+
deployment_id: "UUID",
|
60
|
+
) -> List[str]:
|
61
|
+
"""Gets all arguments that the entrypoint command should be called with.
|
62
|
+
|
63
|
+
Args:
|
64
|
+
run_name: Name of the ZenML run.
|
65
|
+
deployment_id: ID of the deployment.
|
66
|
+
|
67
|
+
Returns:
|
68
|
+
List of entrypoint arguments.
|
69
|
+
"""
|
70
|
+
args = [
|
71
|
+
f"--{RUN_NAME_OPTION}",
|
72
|
+
run_name,
|
73
|
+
f"--{DEPLOYMENT_ID_OPTION}",
|
74
|
+
str(deployment_id),
|
75
|
+
]
|
76
|
+
|
77
|
+
return args
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2023. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Utility functions for the Lightning orchestrator."""
|
15
|
+
|
16
|
+
import itertools
|
17
|
+
import re
|
18
|
+
from typing import List
|
19
|
+
|
20
|
+
from zenml.client import Client
|
21
|
+
from zenml.config import DockerSettings
|
22
|
+
from zenml.utils.pipeline_docker_image_builder import (
|
23
|
+
PipelineDockerImageBuilder,
|
24
|
+
)
|
25
|
+
|
26
|
+
|
27
|
+
def sanitize_studio_name(studio_name: str) -> str:
|
28
|
+
"""Sanitize studio_names so they conform to Kubernetes studio naming convention.
|
29
|
+
|
30
|
+
Args:
|
31
|
+
studio_name: Arbitrary input studio_name.
|
32
|
+
|
33
|
+
Returns:
|
34
|
+
Sanitized pod name.
|
35
|
+
"""
|
36
|
+
studio_name = re.sub(r"[^a-z0-9-]", "-", studio_name.lower())
|
37
|
+
studio_name = re.sub(r"^[-]+", "", studio_name)
|
38
|
+
return re.sub(r"[-]+", "-", studio_name)
|
39
|
+
|
40
|
+
|
41
|
+
def gather_requirements(docker_settings: "DockerSettings") -> List[str]:
|
42
|
+
"""Gather the requirements files.
|
43
|
+
|
44
|
+
Args:
|
45
|
+
docker_settings: Docker settings.
|
46
|
+
|
47
|
+
Returns:
|
48
|
+
List of requirements.
|
49
|
+
"""
|
50
|
+
docker_image_builder = PipelineDockerImageBuilder()
|
51
|
+
requirements_files = docker_image_builder.gather_requirements_files(
|
52
|
+
docker_settings=docker_settings,
|
53
|
+
stack=Client().active_stack,
|
54
|
+
log=False,
|
55
|
+
)
|
56
|
+
|
57
|
+
# Extract and clean the requirements
|
58
|
+
requirements = list(
|
59
|
+
itertools.chain.from_iterable(
|
60
|
+
r[1].strip().split("\n") for r in requirements_files
|
61
|
+
)
|
62
|
+
)
|
63
|
+
|
64
|
+
# Remove empty items and duplicates
|
65
|
+
requirements = sorted(set(filter(None, requirements)))
|
66
|
+
|
67
|
+
return requirements
|
@@ -16,16 +16,21 @@
|
|
16
16
|
The MLflow integrations currently enables you to use MLflow tracking as a
|
17
17
|
convenient way to visualize your experiment runs within the MLflow UI.
|
18
18
|
"""
|
19
|
-
from typing import List, Type
|
19
|
+
from typing import List, Type, Optional
|
20
20
|
|
21
21
|
from zenml.integrations.constants import MLFLOW
|
22
22
|
from zenml.integrations.integration import Integration
|
23
23
|
from zenml.stack import Flavor
|
24
|
+
import sys
|
25
|
+
|
26
|
+
from zenml.logger import get_logger
|
24
27
|
|
25
28
|
MLFLOW_MODEL_DEPLOYER_FLAVOR = "mlflow"
|
26
29
|
MLFLOW_MODEL_EXPERIMENT_TRACKER_FLAVOR = "mlflow"
|
27
30
|
MLFLOW_MODEL_REGISTRY_FLAVOR = "mlflow"
|
28
31
|
|
32
|
+
logger = get_logger(__name__)
|
33
|
+
|
29
34
|
|
30
35
|
class MlflowIntegration(Integration):
|
31
36
|
"""Definition of MLflow integration for ZenML."""
|
@@ -34,8 +39,6 @@ class MlflowIntegration(Integration):
|
|
34
39
|
|
35
40
|
REQUIREMENTS = [
|
36
41
|
"mlflow>=2.1.1,<3",
|
37
|
-
"mlserver>=1.3.3",
|
38
|
-
"mlserver-mlflow>=1.3.3",
|
39
42
|
# TODO: remove this requirement once rapidjson is fixed
|
40
43
|
"python-rapidjson<1.15",
|
41
44
|
# When you do:
|
@@ -45,10 +48,45 @@ class MlflowIntegration(Integration):
|
|
45
48
|
# This downgrades pydantic to v1 even though mlflow does not have
|
46
49
|
# any issues with v2. This is why we have to pin it here so a downgrade
|
47
50
|
# will not happen.
|
48
|
-
"pydantic>=2.
|
51
|
+
"pydantic>=2.8.0,<2.9.0",
|
52
|
+
]
|
53
|
+
|
54
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = [
|
55
|
+
"python-rapidjson",
|
56
|
+
"pydantic",
|
57
|
+
"numpy",
|
58
|
+
"pandas",
|
49
59
|
]
|
50
60
|
|
51
|
-
|
61
|
+
@classmethod
|
62
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
63
|
+
"""Method to get the requirements for the integration.
|
64
|
+
|
65
|
+
Args:
|
66
|
+
target_os: The target operating system to get the requirements for.
|
67
|
+
|
68
|
+
Returns:
|
69
|
+
A list of requirements.
|
70
|
+
"""
|
71
|
+
from zenml.integrations.numpy import NumpyIntegration
|
72
|
+
from zenml.integrations.pandas import PandasIntegration
|
73
|
+
|
74
|
+
reqs = cls.REQUIREMENTS
|
75
|
+
if sys.version_info.minor >= 12:
|
76
|
+
logger.debug(
|
77
|
+
"The MLflow integration on Python 3.12 and above is not yet "
|
78
|
+
"fully supported: The extra dependencies 'mlserver' and "
|
79
|
+
"'mlserver-mlflow' will be skipped."
|
80
|
+
)
|
81
|
+
else:
|
82
|
+
reqs.extend([
|
83
|
+
"mlserver>=1.3.3",
|
84
|
+
"mlserver-mlflow>=1.3.3",
|
85
|
+
])
|
86
|
+
|
87
|
+
reqs.extend(NumpyIntegration.get_requirements(target_os=target_os))
|
88
|
+
reqs.extend(PandasIntegration.get_requirements(target_os=target_os))
|
89
|
+
return reqs
|
52
90
|
|
53
91
|
@classmethod
|
54
92
|
def activate(cls) -> None:
|
@@ -14,6 +14,7 @@
|
|
14
14
|
"""Implementation of the MLflow deployment functionality."""
|
15
15
|
|
16
16
|
import os
|
17
|
+
import sys
|
17
18
|
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
18
19
|
|
19
20
|
import numpy as np
|
@@ -21,6 +22,7 @@ import pandas as pd
|
|
21
22
|
import requests
|
22
23
|
from mlflow.pyfunc.backend import PyFuncBackend
|
23
24
|
from mlflow.version import VERSION as MLFLOW_VERSION
|
25
|
+
from pydantic import field_validator
|
24
26
|
|
25
27
|
from zenml.client import Client
|
26
28
|
from zenml.constants import DEFAULT_SERVICE_START_STOP_TIMEOUT
|
@@ -110,6 +112,30 @@ class MLFlowDeploymentConfig(LocalDaemonServiceConfig):
|
|
110
112
|
mlserver: bool = False
|
111
113
|
timeout: int = DEFAULT_SERVICE_START_STOP_TIMEOUT
|
112
114
|
|
115
|
+
@field_validator("mlserver")
|
116
|
+
@classmethod
|
117
|
+
def validate_mlserver_python_version(cls, mlserver: bool) -> bool:
|
118
|
+
"""Validates the Python version if mlserver is used.
|
119
|
+
|
120
|
+
Args:
|
121
|
+
mlserver: set to True if the MLflow MLServer backend is used,
|
122
|
+
else set to False and MLflow built-in scoring server will be
|
123
|
+
used.
|
124
|
+
|
125
|
+
Returns:
|
126
|
+
the validated value
|
127
|
+
|
128
|
+
Raises:
|
129
|
+
ValueError: if mlserver is set to true on Python 3.12 as it is not
|
130
|
+
yet supported.
|
131
|
+
"""
|
132
|
+
if mlserver is True and sys.version_info.minor >= 12:
|
133
|
+
raise ValueError(
|
134
|
+
"The mlserver deployment is not yet supported on Python 3.12 "
|
135
|
+
"or above."
|
136
|
+
)
|
137
|
+
return mlserver
|
138
|
+
|
113
139
|
|
114
140
|
class MLFlowDeploymentService(LocalDaemonService, BaseDeploymentService):
|
115
141
|
"""MLflow deployment service used to start a local prediction server for MLflow models.
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Initialization of the Numpy integration."""
|
15
|
+
|
16
|
+
from zenml.integrations.constants import NUMPY
|
17
|
+
from zenml.integrations.integration import Integration
|
18
|
+
|
19
|
+
|
20
|
+
class NumpyIntegration(Integration):
|
21
|
+
"""Definition of Numpy integration for ZenML."""
|
22
|
+
|
23
|
+
NAME = NUMPY
|
24
|
+
REQUIREMENTS = ["numpy<2.0.0"]
|
25
|
+
|
26
|
+
@classmethod
|
27
|
+
def activate(cls) -> None:
|
28
|
+
"""Activates the integration."""
|
29
|
+
from zenml.integrations.numpy import materializers # noqa
|
30
|
+
|
31
|
+
|
32
|
+
NumpyIntegration.check_installation()
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Initialization of the Numpy materializer."""
|
15
|
+
|
16
|
+
from zenml.integrations.numpy.materializers.numpy_materializer import ( # noqa
|
17
|
+
NumpyMaterializer,
|
18
|
+
)
|