zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
zenml/models/__init__.py
CHANGED
@@ -332,7 +332,6 @@ from zenml.models.v2.misc.service_connector_type import (
|
|
332
332
|
ResourceTypeModel,
|
333
333
|
)
|
334
334
|
from zenml.models.v2.misc.server_models import ServerDatabaseType, ServerModel
|
335
|
-
from zenml.models.v2.misc.full_stack import FullStackRequest
|
336
335
|
from zenml.models.v2.core.trigger import (
|
337
336
|
TriggerRequest,
|
338
337
|
TriggerFilter,
|
@@ -390,6 +389,12 @@ from zenml.models.v2.misc.stack_deployment import (
|
|
390
389
|
StackDeploymentConfig,
|
391
390
|
StackDeploymentInfo,
|
392
391
|
)
|
392
|
+
from zenml.models.v2.misc.info_models import (
|
393
|
+
ComponentInfo,
|
394
|
+
ServiceConnectorInfo,
|
395
|
+
ServiceConnectorResourcesInfo,
|
396
|
+
ResourcesInfo,
|
397
|
+
)
|
393
398
|
|
394
399
|
# ----------------------------- Forward References -----------------------------
|
395
400
|
|
@@ -409,7 +414,6 @@ EventSourceResponseMetadata.model_rebuild()
|
|
409
414
|
EventSourceResponseResources.model_rebuild()
|
410
415
|
FlavorResponseBody.model_rebuild()
|
411
416
|
FlavorResponseMetadata.model_rebuild()
|
412
|
-
FullStackRequest.model_rebuild()
|
413
417
|
LazyArtifactVersionResponse.model_rebuild()
|
414
418
|
LazyRunMetadataResponse.model_rebuild()
|
415
419
|
ModelResponseBody.model_rebuild()
|
@@ -448,8 +452,10 @@ ServiceResponseMetadata.model_rebuild()
|
|
448
452
|
ServiceResponseResources.model_rebuild()
|
449
453
|
ServiceConnectorResponseBody.model_rebuild()
|
450
454
|
ServiceConnectorResponseMetadata.model_rebuild()
|
455
|
+
StackRequest.model_rebuild()
|
451
456
|
StackResponseBody.model_rebuild()
|
452
457
|
StackResponseMetadata.model_rebuild()
|
458
|
+
StackResponse.model_rebuild()
|
453
459
|
StepRunRequest.model_rebuild()
|
454
460
|
StepRunResponseBody.model_rebuild()
|
455
461
|
StepRunResponseMetadata.model_rebuild()
|
@@ -458,6 +464,10 @@ TriggerResponseBody.model_rebuild()
|
|
458
464
|
TriggerResponseMetadata.model_rebuild()
|
459
465
|
TriggerResponseResources.model_rebuild()
|
460
466
|
TriggerResponseResources.model_rebuild()
|
467
|
+
ComponentInfo.model_rebuild()
|
468
|
+
ServiceConnectorInfo.model_rebuild()
|
469
|
+
ServiceConnectorResourcesInfo.model_rebuild()
|
470
|
+
ResourcesInfo.model_rebuild()
|
461
471
|
|
462
472
|
|
463
473
|
__all__ = [
|
@@ -723,7 +733,6 @@ __all__ = [
|
|
723
733
|
"ServiceConnectorTypedResourcesModel",
|
724
734
|
"ServiceConnectorRequirements",
|
725
735
|
"ResourceTypeModel",
|
726
|
-
"FullStackRequest",
|
727
736
|
"UserAuthModel",
|
728
737
|
"ExternalUserModel",
|
729
738
|
"BuildItem",
|
@@ -740,4 +749,8 @@ __all__ = [
|
|
740
749
|
"OAuthDeviceVerificationRequest",
|
741
750
|
"OAuthRedirectResponse",
|
742
751
|
"OAuthTokenResponse",
|
752
|
+
"ComponentInfo",
|
753
|
+
"ServiceConnectorInfo",
|
754
|
+
"ServiceConnectorResourcesInfo",
|
755
|
+
"ResourcesInfo",
|
743
756
|
]
|
@@ -343,8 +343,8 @@ class ModelVersionResponse(
|
|
343
343
|
version=self.name,
|
344
344
|
was_created_in_this_run=was_created_in_this_run,
|
345
345
|
suppress_class_validation_warnings=suppress_class_validation_warnings,
|
346
|
+
model_version_id=self.id,
|
346
347
|
)
|
347
|
-
mv.model_version_id = self.id
|
348
348
|
|
349
349
|
return mv
|
350
350
|
|
@@ -115,6 +115,13 @@ class PipelineRunRequest(WorkspaceScopedRequest):
|
|
115
115
|
default=None,
|
116
116
|
title="Tags of the pipeline run.",
|
117
117
|
)
|
118
|
+
model_version_id: Optional[UUID] = Field(
|
119
|
+
title="The ID of the model version that was "
|
120
|
+
"configured by this pipeline run explicitly.",
|
121
|
+
default=None,
|
122
|
+
)
|
123
|
+
|
124
|
+
model_config = ConfigDict(protected_namespaces=())
|
118
125
|
|
119
126
|
|
120
127
|
# ------------------ Update Model ------------------
|
@@ -125,7 +132,12 @@ class PipelineRunUpdate(BaseModel):
|
|
125
132
|
|
126
133
|
status: Optional[ExecutionStatus] = None
|
127
134
|
end_time: Optional[datetime] = None
|
128
|
-
|
135
|
+
model_version_id: Optional[UUID] = Field(
|
136
|
+
title="The ID of the model version that was "
|
137
|
+
"configured by this pipeline run explicitly.",
|
138
|
+
default=None,
|
139
|
+
)
|
140
|
+
# TODO: we should maybe have a different update model here, the upper three attributes should only be for internal use
|
129
141
|
add_tags: Optional[List[str]] = Field(
|
130
142
|
default=None, title="New tags to add to the pipeline run."
|
131
143
|
)
|
@@ -133,6 +145,8 @@ class PipelineRunUpdate(BaseModel):
|
|
133
145
|
default=None, title="Tags to remove from the pipeline run."
|
134
146
|
)
|
135
147
|
|
148
|
+
model_config = ConfigDict(protected_namespaces=())
|
149
|
+
|
136
150
|
|
137
151
|
# ------------------ Response Model ------------------
|
138
152
|
|
@@ -164,6 +178,13 @@ class PipelineRunResponseBody(WorkspaceScopedResponseBody):
|
|
164
178
|
trigger_execution: Optional["TriggerExecutionResponse"] = Field(
|
165
179
|
default=None, title="The trigger execution that triggered this run."
|
166
180
|
)
|
181
|
+
model_version_id: Optional[UUID] = Field(
|
182
|
+
title="The ID of the model version that was "
|
183
|
+
"configured by this pipeline run explicitly.",
|
184
|
+
default=None,
|
185
|
+
)
|
186
|
+
|
187
|
+
model_config = ConfigDict(protected_namespaces=())
|
167
188
|
|
168
189
|
|
169
190
|
class PipelineRunResponseMetadata(WorkspaceScopedResponseMetadata):
|
@@ -357,6 +378,15 @@ class PipelineRunResponse(
|
|
357
378
|
"""
|
358
379
|
return self.get_body().deployment_id
|
359
380
|
|
381
|
+
@property
|
382
|
+
def model_version_id(self) -> Optional[UUID]:
|
383
|
+
"""The `model_version_id` property.
|
384
|
+
|
385
|
+
Returns:
|
386
|
+
the value of the property.
|
387
|
+
"""
|
388
|
+
return self.get_body().model_version_id
|
389
|
+
|
360
390
|
@property
|
361
391
|
def run_metadata(self) -> Dict[str, "RunMetadataResponse"]:
|
362
392
|
"""The `run_metadata` property.
|
zenml/models/v2/core/stack.py
CHANGED
@@ -17,31 +17,38 @@ import json
|
|
17
17
|
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union
|
18
18
|
from uuid import UUID
|
19
19
|
|
20
|
-
from pydantic import Field
|
20
|
+
from pydantic import Field, model_validator
|
21
21
|
from sqlmodel import and_
|
22
22
|
|
23
23
|
from zenml.constants import STR_FIELD_MAX_LENGTH
|
24
24
|
from zenml.enums import StackComponentType
|
25
|
-
from zenml.models.v2.base.base import BaseUpdate
|
25
|
+
from zenml.models.v2.base.base import BaseRequest, BaseUpdate
|
26
26
|
from zenml.models.v2.base.scoped import (
|
27
27
|
WorkspaceScopedFilter,
|
28
|
-
WorkspaceScopedRequest,
|
29
28
|
WorkspaceScopedResponse,
|
30
29
|
WorkspaceScopedResponseBody,
|
31
30
|
WorkspaceScopedResponseMetadata,
|
32
31
|
WorkspaceScopedResponseResources,
|
33
32
|
)
|
34
|
-
from zenml.models.v2.
|
33
|
+
from zenml.models.v2.misc.info_models import (
|
34
|
+
ComponentInfo,
|
35
|
+
ServiceConnectorInfo,
|
36
|
+
)
|
35
37
|
|
36
38
|
if TYPE_CHECKING:
|
37
39
|
from sqlalchemy.sql.elements import ColumnElement
|
38
40
|
|
41
|
+
from zenml.models.v2.core.component import ComponentResponse
|
42
|
+
|
39
43
|
|
40
44
|
# ------------------ Request Model ------------------
|
41
45
|
|
42
46
|
|
43
|
-
class StackRequest(
|
44
|
-
"""Request model for
|
47
|
+
class StackRequest(BaseRequest):
|
48
|
+
"""Request model for a stack."""
|
49
|
+
|
50
|
+
user: Optional[UUID] = None
|
51
|
+
workspace: Optional[UUID] = None
|
45
52
|
|
46
53
|
name: str = Field(
|
47
54
|
title="The name of the stack.", max_length=STR_FIELD_MAX_LENGTH
|
@@ -55,15 +62,26 @@ class StackRequest(WorkspaceScopedRequest):
|
|
55
62
|
default=None,
|
56
63
|
title="The path to the stack spec used for mlstacks deployments.",
|
57
64
|
)
|
58
|
-
components:
|
59
|
-
|
60
|
-
|
61
|
-
|
65
|
+
components: Dict[StackComponentType, List[Union[UUID, ComponentInfo]]] = (
|
66
|
+
Field(
|
67
|
+
title="The mapping for the components of the full stack registration.",
|
68
|
+
description="The mapping from component types to either UUIDs of "
|
69
|
+
"existing components or request information for brand new "
|
70
|
+
"components.",
|
71
|
+
)
|
62
72
|
)
|
63
73
|
labels: Optional[Dict[str, Any]] = Field(
|
64
74
|
default=None,
|
65
75
|
title="The stack labels.",
|
66
76
|
)
|
77
|
+
service_connectors: List[Union[UUID, ServiceConnectorInfo]] = Field(
|
78
|
+
default=[],
|
79
|
+
title="The service connectors dictionary for the full stack "
|
80
|
+
"registration.",
|
81
|
+
description="The UUID of an already existing service connector or "
|
82
|
+
"request information to create a service connector from "
|
83
|
+
"scratch.",
|
84
|
+
)
|
67
85
|
|
68
86
|
@property
|
69
87
|
def is_valid(self) -> bool:
|
@@ -79,14 +97,25 @@ class StackRequest(WorkspaceScopedRequest):
|
|
79
97
|
and StackComponentType.ORCHESTRATOR in self.components
|
80
98
|
)
|
81
99
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
100
|
+
@model_validator(mode="after")
|
101
|
+
def _validate_indexes_in_components(self) -> "StackRequest":
|
102
|
+
for components in self.components.values():
|
103
|
+
for component in components:
|
104
|
+
if isinstance(component, ComponentInfo):
|
105
|
+
if component.service_connector_index is not None:
|
106
|
+
if (
|
107
|
+
component.service_connector_index < 0
|
108
|
+
or component.service_connector_index
|
109
|
+
>= len(self.service_connectors)
|
110
|
+
):
|
111
|
+
raise ValueError(
|
112
|
+
f"Service connector index "
|
113
|
+
f"{component.service_connector_index} "
|
114
|
+
"is out of range. Please provide a valid index "
|
115
|
+
"referring to the position in the list of service "
|
116
|
+
"connectors."
|
117
|
+
)
|
118
|
+
return self
|
90
119
|
|
91
120
|
|
92
121
|
# ------------------ Update Model ------------------
|
@@ -130,7 +159,7 @@ class StackResponseBody(WorkspaceScopedResponseBody):
|
|
130
159
|
class StackResponseMetadata(WorkspaceScopedResponseMetadata):
|
131
160
|
"""Response metadata for stacks."""
|
132
161
|
|
133
|
-
components: Dict[StackComponentType, List[ComponentResponse]] = Field(
|
162
|
+
components: Dict[StackComponentType, List["ComponentResponse"]] = Field(
|
134
163
|
title="A mapping of stack component types to the actual"
|
135
164
|
"instances of components of this type."
|
136
165
|
)
|
@@ -257,7 +286,9 @@ class StackResponse(
|
|
257
286
|
return self.get_metadata().stack_spec_path
|
258
287
|
|
259
288
|
@property
|
260
|
-
def components(
|
289
|
+
def components(
|
290
|
+
self,
|
291
|
+
) -> Dict[StackComponentType, List["ComponentResponse"]]:
|
261
292
|
"""The `components` property.
|
262
293
|
|
263
294
|
Returns:
|
zenml/models/v2/core/step_run.py
CHANGED
@@ -108,6 +108,13 @@ class StepRunRequest(WorkspaceScopedRequest):
|
|
108
108
|
deployment: UUID = Field(
|
109
109
|
title="The deployment associated with the step run."
|
110
110
|
)
|
111
|
+
model_version_id: Optional[UUID] = Field(
|
112
|
+
title="The ID of the model version that was "
|
113
|
+
"configured by this step run explicitly.",
|
114
|
+
default=None,
|
115
|
+
)
|
116
|
+
|
117
|
+
model_config = ConfigDict(protected_namespaces=())
|
111
118
|
|
112
119
|
|
113
120
|
# ------------------ Update Model ------------------
|
@@ -136,6 +143,12 @@ class StepRunUpdate(BaseModel):
|
|
136
143
|
title="The end time of the step run.",
|
137
144
|
default=None,
|
138
145
|
)
|
146
|
+
model_version_id: Optional[UUID] = Field(
|
147
|
+
title="The ID of the model version that was "
|
148
|
+
"configured by this step run explicitly.",
|
149
|
+
default=None,
|
150
|
+
)
|
151
|
+
model_config = ConfigDict(protected_namespaces=())
|
139
152
|
|
140
153
|
|
141
154
|
# ------------------ Response Model ------------------
|
@@ -151,6 +164,12 @@ class StepRunResponseBody(WorkspaceScopedResponseBody):
|
|
151
164
|
title="The output artifact versions of the step run.",
|
152
165
|
default={},
|
153
166
|
)
|
167
|
+
model_version_id: Optional[UUID] = Field(
|
168
|
+
title="The ID of the model version that was "
|
169
|
+
"configured by this step run explicitly.",
|
170
|
+
default=None,
|
171
|
+
)
|
172
|
+
model_config = ConfigDict(protected_namespaces=())
|
154
173
|
|
155
174
|
|
156
175
|
class StepRunResponseMetadata(WorkspaceScopedResponseMetadata):
|
@@ -320,6 +339,15 @@ class StepRunResponse(
|
|
320
339
|
"""
|
321
340
|
return self.get_body().outputs
|
322
341
|
|
342
|
+
@property
|
343
|
+
def model_version_id(self) -> Optional[UUID]:
|
344
|
+
"""The `model_version_id` property.
|
345
|
+
|
346
|
+
Returns:
|
347
|
+
the value of the property.
|
348
|
+
"""
|
349
|
+
return self.get_body().model_version_id
|
350
|
+
|
323
351
|
@property
|
324
352
|
def config(self) -> "StepConfiguration":
|
325
353
|
"""The `config` property.
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Models representing full stack requests."""
|
15
|
+
|
16
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
17
|
+
|
18
|
+
from pydantic import BaseModel, Field, model_validator
|
19
|
+
|
20
|
+
from zenml.enums import StackComponentType
|
21
|
+
|
22
|
+
if TYPE_CHECKING:
|
23
|
+
from zenml.models.v2.core.component import ComponentResponse
|
24
|
+
|
25
|
+
|
26
|
+
class ServiceConnectorInfo(BaseModel):
|
27
|
+
"""Information about the service connector when creating a full stack."""
|
28
|
+
|
29
|
+
type: str
|
30
|
+
auth_method: str
|
31
|
+
configuration: Dict[str, Any] = {}
|
32
|
+
|
33
|
+
|
34
|
+
class ComponentInfo(BaseModel):
|
35
|
+
"""Information about each stack components when creating a full stack."""
|
36
|
+
|
37
|
+
flavor: str
|
38
|
+
service_connector_index: Optional[int] = Field(
|
39
|
+
default=None,
|
40
|
+
title="The id of the service connector from the list "
|
41
|
+
"`service_connectors`.",
|
42
|
+
description="The id of the service connector from the list "
|
43
|
+
"`service_connectors` from `FullStackRequest`.",
|
44
|
+
)
|
45
|
+
service_connector_resource_id: Optional[str] = None
|
46
|
+
configuration: Dict[str, Any] = {}
|
47
|
+
|
48
|
+
|
49
|
+
class ResourcesInfo(BaseModel):
|
50
|
+
"""Information about the resources needed for CLI and UI."""
|
51
|
+
|
52
|
+
flavor: str
|
53
|
+
flavor_display_name: str
|
54
|
+
required_configuration: Dict[str, str] = {}
|
55
|
+
use_resource_value_as_fixed_config: bool = False
|
56
|
+
|
57
|
+
accessible_by_service_connector: List[str]
|
58
|
+
connected_through_service_connector: List["ComponentResponse"]
|
59
|
+
|
60
|
+
@model_validator(mode="after")
|
61
|
+
def _validate_resource_info(self) -> "ResourcesInfo":
|
62
|
+
if (
|
63
|
+
self.use_resource_value_as_fixed_config
|
64
|
+
and len(self.required_configuration) > 1
|
65
|
+
):
|
66
|
+
raise ValueError(
|
67
|
+
"Cannot use resource value as fixed config if more than "
|
68
|
+
"one required configuration key is provided."
|
69
|
+
)
|
70
|
+
return self
|
71
|
+
|
72
|
+
|
73
|
+
class ServiceConnectorResourcesInfo(BaseModel):
|
74
|
+
"""Information about the service connector resources needed for CLI and UI."""
|
75
|
+
|
76
|
+
connector_type: str
|
77
|
+
|
78
|
+
components_resources_info: Dict[StackComponentType, List[ResourcesInfo]]
|
zenml/new/pipelines/pipeline.py
CHANGED
@@ -17,7 +17,6 @@ import copy
|
|
17
17
|
import hashlib
|
18
18
|
import inspect
|
19
19
|
from contextlib import contextmanager
|
20
|
-
from datetime import datetime
|
21
20
|
from pathlib import Path
|
22
21
|
from typing import (
|
23
22
|
TYPE_CHECKING,
|
@@ -72,7 +71,6 @@ from zenml.new.pipelines import build_utils
|
|
72
71
|
from zenml.new.pipelines.run_utils import (
|
73
72
|
create_placeholder_run,
|
74
73
|
deploy_pipeline,
|
75
|
-
prepare_model_versions,
|
76
74
|
upload_notebook_cell_code_if_necessary,
|
77
75
|
)
|
78
76
|
from zenml.stack import Stack
|
@@ -91,6 +89,7 @@ from zenml.utils import (
|
|
91
89
|
source_utils,
|
92
90
|
yaml_utils,
|
93
91
|
)
|
92
|
+
from zenml.utils.string_utils import format_name_template
|
94
93
|
|
95
94
|
if TYPE_CHECKING:
|
96
95
|
from zenml.artifacts.external_artifact import ExternalArtifact
|
@@ -587,6 +586,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
587
586
|
Returns:
|
588
587
|
Model of the pipeline run if running without a schedule, `None` if
|
589
588
|
running with a schedule.
|
589
|
+
|
590
|
+
Raises:
|
591
|
+
ValueError: if the orchestrator doesn't support scheduling, but schedule was given
|
590
592
|
"""
|
591
593
|
if constants.SHOULD_PREVENT_PIPELINE_EXECUTION:
|
592
594
|
# An environment variable was set to stop the execution of
|
@@ -632,17 +634,24 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
632
634
|
else:
|
633
635
|
logger.debug(f"Pipeline {self.name} is unlisted.")
|
634
636
|
|
635
|
-
|
636
|
-
|
637
|
+
stack = Client().active_stack
|
638
|
+
stack.validate()
|
639
|
+
|
637
640
|
schedule_id = None
|
638
641
|
if schedule:
|
642
|
+
if not stack.orchestrator.config.is_schedulable:
|
643
|
+
raise ValueError(
|
644
|
+
f"Stack {stack.name} does not support scheduling. "
|
645
|
+
"Not all orchestrator types support scheduling, "
|
646
|
+
"kindly consult with "
|
647
|
+
"https://docs.zenml.io/how-to/build-pipelines/schedule-a-pipeline "
|
648
|
+
"for details."
|
649
|
+
)
|
639
650
|
if schedule.name:
|
640
651
|
schedule_name = schedule.name
|
641
652
|
else:
|
642
|
-
|
643
|
-
|
644
|
-
schedule_name = deployment.run_name_template.format(
|
645
|
-
date=date, time=time
|
653
|
+
schedule_name = format_name_template(
|
654
|
+
deployment.run_name_template
|
646
655
|
)
|
647
656
|
components = Client().active_stack_model.components
|
648
657
|
orchestrator = components[StackComponentType.ORCHESTRATOR][0]
|
@@ -674,8 +683,6 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
674
683
|
deployment=deployment, stack=stack
|
675
684
|
)
|
676
685
|
|
677
|
-
prepare_model_versions(deployment)
|
678
|
-
|
679
686
|
local_repo_context = (
|
680
687
|
code_repository_utils.find_active_code_repository()
|
681
688
|
)
|
@@ -753,7 +760,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
753
760
|
if run:
|
754
761
|
run_url = dashboard_utils.get_run_url(run)
|
755
762
|
if run_url:
|
756
|
-
logger.info(f"Dashboard URL: {run_url}")
|
763
|
+
logger.info(f"Dashboard URL for Pipeline Run: {run_url}")
|
757
764
|
else:
|
758
765
|
logger.info(
|
759
766
|
"You can visualize your pipeline runs in the `ZenML "
|
@@ -1009,12 +1016,14 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
1009
1016
|
|
1010
1017
|
integration_registry.activate_integrations()
|
1011
1018
|
|
1012
|
-
self._parse_config_file(
|
1019
|
+
_from_config_file = self._parse_config_file(
|
1013
1020
|
config_path=config_path,
|
1014
1021
|
matcher=list(PipelineRunConfiguration.model_fields.keys()),
|
1015
1022
|
)
|
1016
1023
|
|
1017
|
-
|
1024
|
+
self._reconfigure_from_file_with_overrides(config_path=config_path)
|
1025
|
+
|
1026
|
+
run_config = PipelineRunConfiguration(**_from_config_file)
|
1018
1027
|
|
1019
1028
|
new_values = dict_utils.remove_none_values(run_configuration_args)
|
1020
1029
|
update = PipelineRunConfiguration.model_validate(new_values)
|
@@ -1229,12 +1238,15 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
1229
1238
|
|
1230
1239
|
def _parse_config_file(
|
1231
1240
|
self, config_path: Optional[str], matcher: List[str]
|
1232
|
-
) ->
|
1241
|
+
) -> Dict[str, Any]:
|
1233
1242
|
"""Parses the given configuration file and sets `self._from_config_file`.
|
1234
1243
|
|
1235
1244
|
Args:
|
1236
1245
|
config_path: Path to a yaml configuration file.
|
1237
1246
|
matcher: List of keys to match in the configuration file.
|
1247
|
+
|
1248
|
+
Returns:
|
1249
|
+
Parsed config file according to matcher settings.
|
1238
1250
|
"""
|
1239
1251
|
_from_config_file: Dict[str, Any] = {}
|
1240
1252
|
if config_path:
|
@@ -1263,7 +1275,7 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
1263
1275
|
_from_config_file["model"] = Model.model_validate(
|
1264
1276
|
_from_config_file["model"]
|
1265
1277
|
)
|
1266
|
-
|
1278
|
+
return _from_config_file
|
1267
1279
|
|
1268
1280
|
def with_options(
|
1269
1281
|
self,
|
@@ -1314,17 +1326,9 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
1314
1326
|
|
1315
1327
|
pipeline_copy = self.copy()
|
1316
1328
|
|
1317
|
-
pipeline_copy.
|
1318
|
-
config_path=config_path,
|
1319
|
-
matcher=inspect.getfullargspec(self.configure)[0]
|
1320
|
-
+ ["model_version"], # TODO: deprecate `model_version` later on
|
1329
|
+
pipeline_copy._reconfigure_from_file_with_overrides(
|
1330
|
+
config_path=config_path, **kwargs
|
1321
1331
|
)
|
1322
|
-
pipeline_copy._from_config_file = dict_utils.recursive_update(
|
1323
|
-
pipeline_copy._from_config_file, kwargs
|
1324
|
-
)
|
1325
|
-
|
1326
|
-
with pipeline_copy.__suppress_configure_warnings__():
|
1327
|
-
pipeline_copy.configure(**pipeline_copy._from_config_file)
|
1328
1332
|
|
1329
1333
|
run_args = dict_utils.remove_none_values(
|
1330
1334
|
{
|
@@ -1422,3 +1426,39 @@ To avoid this consider setting pipeline parameters only in one place (config or
|
|
1422
1426
|
)
|
1423
1427
|
else:
|
1424
1428
|
self.prepare()
|
1429
|
+
|
1430
|
+
def _reconfigure_from_file_with_overrides(
|
1431
|
+
self,
|
1432
|
+
config_path: Optional[str] = None,
|
1433
|
+
**kwargs: Any,
|
1434
|
+
) -> None:
|
1435
|
+
"""Update the pipeline configuration from config file.
|
1436
|
+
|
1437
|
+
Accepts overrides as kwargs.
|
1438
|
+
|
1439
|
+
Args:
|
1440
|
+
config_path: Path to a yaml configuration file. This file will
|
1441
|
+
be parsed as a
|
1442
|
+
`zenml.config.pipeline_configurations.PipelineRunConfiguration`
|
1443
|
+
object. Options provided in this file will be overwritten by
|
1444
|
+
options provided in code using the other arguments of this
|
1445
|
+
method.
|
1446
|
+
**kwargs: Pipeline configuration options. These will be passed
|
1447
|
+
to the `pipeline.configure(...)` method.
|
1448
|
+
"""
|
1449
|
+
self._from_config_file = {}
|
1450
|
+
if config_path:
|
1451
|
+
self._from_config_file = self._parse_config_file(
|
1452
|
+
config_path=config_path,
|
1453
|
+
matcher=inspect.getfullargspec(self.configure)[0]
|
1454
|
+
+ [
|
1455
|
+
"model_version"
|
1456
|
+
], # TODO: deprecate `model_version` later on
|
1457
|
+
)
|
1458
|
+
|
1459
|
+
_from_config_file = dict_utils.recursive_update(
|
1460
|
+
self._from_config_file, kwargs
|
1461
|
+
)
|
1462
|
+
|
1463
|
+
with self.__suppress_configure_warnings__():
|
1464
|
+
self.configure(**_from_config_file)
|