zenml-nightly 0.66.0.dev20240928__py3-none-any.whl → 0.68.1.dev20241102__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +18 -12
- RELEASE_NOTES.md +236 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +5 -10
- zenml/analytics/context.py +16 -1
- zenml/analytics/enums.py +0 -13
- zenml/analytics/utils.py +18 -7
- zenml/artifact_stores/base_artifact_store.py +11 -0
- zenml/artifacts/artifact_config.py +19 -65
- zenml/artifacts/external_artifact.py +7 -53
- zenml/artifacts/external_artifact_config.py +21 -40
- zenml/artifacts/load_directory_materializer.py +91 -0
- zenml/artifacts/utils.py +204 -145
- zenml/cli/__init__.py +62 -127
- zenml/cli/base.py +8 -8
- zenml/cli/integration.py +0 -9
- zenml/cli/login.py +951 -0
- zenml/cli/pipeline.py +1 -1
- zenml/cli/served_model.py +8 -1
- zenml/cli/server.py +462 -577
- zenml/cli/service_accounts.py +4 -4
- zenml/cli/stack.py +78 -569
- zenml/cli/stack_components.py +5 -594
- zenml/cli/user_management.py +0 -12
- zenml/cli/utils.py +24 -142
- zenml/client.py +57 -40
- zenml/client_lazy_loader.py +29 -12
- zenml/config/compiler.py +7 -11
- zenml/config/global_config.py +9 -0
- zenml/config/pipeline_configurations.py +2 -1
- zenml/config/pipeline_run_configuration.py +2 -1
- zenml/config/server_config.py +7 -5
- zenml/config/step_configurations.py +2 -0
- zenml/constants.py +6 -42
- zenml/enums.py +1 -4
- zenml/environment.py +1 -315
- zenml/exceptions.py +12 -41
- zenml/hooks/hook_validators.py +1 -8
- zenml/integrations/airflow/orchestrators/airflow_orchestrator.py +2 -3
- zenml/integrations/argilla/__init__.py +1 -1
- zenml/integrations/argilla/annotators/argilla_annotator.py +234 -89
- zenml/integrations/argilla/flavors/argilla_annotator_flavor.py +10 -5
- zenml/integrations/aws/flavors/sagemaker_orchestrator_flavor.py +9 -5
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +15 -2
- zenml/integrations/aws/service_connectors/aws_service_connector.py +6 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +5 -4
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
- zenml/integrations/bentoml/model_deployers/bentoml_model_deployer.py +85 -19
- zenml/integrations/bentoml/services/__init__.py +15 -4
- zenml/integrations/bentoml/services/bentoml_container_deployment.py +399 -0
- zenml/integrations/bentoml/services/{bentoml_deployment.py → bentoml_local_deployment.py} +81 -40
- zenml/{steps/base_parameters.py → integrations/bentoml/services/deployment_type.py} +8 -5
- zenml/integrations/bentoml/steps/bento_builder.py +1 -0
- zenml/integrations/bentoml/steps/bentoml_deployer.py +93 -41
- zenml/integrations/github/code_repositories/github_code_repository.py +1 -1
- zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py +6 -1
- zenml/integrations/huggingface/__init__.py +1 -6
- zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +3 -3
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +3 -3
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +2 -2
- zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +6 -3
- zenml/integrations/mlflow/model_registries/mlflow_model_registry.py +6 -0
- zenml/integrations/mlflow/services/mlflow_deployment.py +2 -2
- zenml/integrations/mlflow/steps/mlflow_deployer.py +1 -1
- zenml/integrations/mlflow/steps/mlflow_registry.py +1 -1
- zenml/integrations/neptune/experiment_trackers/run_state.py +2 -2
- zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +8 -2
- zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +4 -1
- zenml/integrations/s3/artifact_stores/s3_artifact_store.py +98 -0
- zenml/integrations/s3/flavors/s3_artifact_store_flavor.py +14 -0
- zenml/integrations/s3/utils.py +40 -0
- zenml/integrations/tensorflow/__init__.py +0 -9
- zenml/logging/step_logging.py +13 -15
- zenml/{new/pipelines → login}/__init__.py +4 -1
- zenml/login/credentials.py +346 -0
- zenml/login/credentials_store.py +603 -0
- zenml/{new → login/pro}/__init__.py +4 -1
- zenml/login/pro/client.py +496 -0
- zenml/login/pro/constants.py +34 -0
- zenml/{lineage_graph/edge.py → login/pro/models.py} +9 -8
- zenml/{zen_server/deploy/terraform/providers → login/pro/organization}/__init__.py +2 -2
- zenml/login/pro/organization/client.py +79 -0
- zenml/{lineage_graph/node/base_node.py → login/pro/organization/models.py} +13 -12
- zenml/{new/steps → login/pro/tenant}/__init__.py +2 -1
- zenml/login/pro/tenant/client.py +92 -0
- zenml/login/pro/tenant/models.py +174 -0
- zenml/login/pro/utils.py +121 -0
- zenml/{cli → login}/web_login.py +64 -28
- zenml/materializers/__init__.py +2 -0
- zenml/materializers/base_materializer.py +43 -9
- zenml/materializers/built_in_materializer.py +1 -1
- zenml/materializers/uuid_materializer.py +79 -0
- zenml/metadata/lazy_load.py +10 -7
- zenml/metadata/metadata_types.py +49 -0
- zenml/model/lazy_load.py +85 -4
- zenml/model/model.py +61 -83
- zenml/model/utils.py +47 -118
- zenml/models/__init__.py +6 -0
- zenml/models/v2/base/base.py +12 -8
- zenml/models/v2/base/filter.py +9 -0
- zenml/models/v2/core/artifact_version.py +53 -25
- zenml/models/v2/core/component.py +54 -19
- zenml/models/v2/core/flavor.py +13 -13
- zenml/models/v2/core/model.py +4 -1
- zenml/models/v2/core/model_version.py +4 -6
- zenml/models/v2/core/model_version_artifact.py +3 -1
- zenml/models/v2/core/model_version_pipeline_run.py +3 -1
- zenml/models/v2/core/pipeline.py +3 -1
- zenml/models/v2/core/pipeline_run.py +24 -3
- zenml/models/v2/core/run_metadata.py +2 -2
- zenml/models/v2/core/run_template.py +3 -1
- zenml/models/v2/core/stack.py +7 -3
- zenml/models/v2/core/step_run.py +58 -14
- zenml/models/v2/misc/auth_models.py +11 -2
- zenml/models/v2/misc/server_models.py +2 -4
- zenml/orchestrators/base_orchestrator.py +41 -4
- zenml/orchestrators/input_utils.py +52 -36
- zenml/orchestrators/step_launcher.py +50 -171
- zenml/orchestrators/step_run_utils.py +574 -0
- zenml/orchestrators/step_runner.py +83 -120
- zenml/orchestrators/utils.py +46 -180
- zenml/pipelines/__init__.py +3 -16
- zenml/{new/pipelines → pipelines}/pipeline_context.py +3 -18
- zenml/pipelines/pipeline_decorator.py +34 -60
- zenml/{new/pipelines/pipeline.py → pipelines/pipeline_definition.py} +210 -207
- zenml/{new/pipelines → pipelines}/run_utils.py +11 -10
- zenml/services/local/local_daemon_entrypoint.py +4 -4
- zenml/services/service.py +2 -2
- zenml/stack/stack.py +3 -108
- zenml/stack/stack_component.py +2 -78
- zenml/stack/utils.py +26 -14
- zenml/stack_deployments/aws_stack_deployment.py +3 -0
- zenml/step_operators/step_operator_entrypoint_configuration.py +1 -1
- zenml/steps/__init__.py +3 -9
- zenml/steps/base_step.py +25 -267
- zenml/steps/entrypoint_function_utils.py +2 -80
- zenml/{new/steps → steps}/step_context.py +12 -66
- zenml/steps/step_decorator.py +42 -89
- zenml/steps/step_invocation.py +16 -60
- zenml/steps/utils.py +4 -23
- zenml/utils/dashboard_utils.py +16 -51
- zenml/utils/materializer_utils.py +1 -1
- zenml/utils/requirements_utils.py +71 -0
- zenml/utils/singleton.py +15 -3
- zenml/utils/source_utils.py +39 -2
- zenml/utils/typing_utils.py +8 -28
- zenml/utils/visualization_utils.py +1 -1
- zenml/zen_server/auth.py +44 -39
- zenml/zen_server/dashboard/assets/{404-Y50hSt65.js → 404-DT4QRUqN.js} +1 -1
- zenml/zen_server/dashboard/assets/@radix-DP6vWzyx.js +85 -0
- zenml/zen_server/dashboard/assets/@react-router-BMhZulnd.js +29 -0
- zenml/zen_server/dashboard/assets/@reactflow-8U9qNlMR.js +17 -0
- zenml/zen_server/dashboard/assets/@tanstack-BUCbhJyH.js +22 -0
- zenml/zen_server/dashboard/assets/{AlertDialogDropdownItem-xLR9a1iw.js → AlertDialogDropdownItem-C6N2rGrB.js} +1 -1
- zenml/zen_server/dashboard/assets/{CodeSnippet-IxXNxUDa.js → CodeSnippet-CqybNv0k.js} +1 -1
- zenml/zen_server/dashboard/assets/{CollapsibleCard-BhutZbBL.js → CollapsibleCard-0r_8G2Lj.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-Bf-rd1z8.js → Commands-BDjgBQKi.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +1 -0
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +1 -0
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-DcFHidFJ.js → CopyButton-C745BrKi.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-QSbjrfxw.js → CsvVizualization-PpAq0CeZ.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-Cd3HqST4.js → DialogItem-DcVCZEew.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DkCy54Bp.js → DisplayDate-BeXgUG_C.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-Cs3DEmso.js → EmptyState-DeK7H4pr.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BhwdmqK7.js → Error-BMlzibXj.js} +1 -1
- zenml/zen_server/dashboard/assets/{Helpbox-0pBpTwTm.js → Helpbox-BLf40fLV.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BTK_EUKT.js → Infobox-BwisKifi.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-CA3DFMcM.js → InlineAvatar-jEgodSgX.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CmIn0szs.js → Lock-3lLt1ih0.js} +1 -1
- zenml/zen_server/dashboard/assets/MarkdownVisualization-8O9kTr-2.js +14 -0
- zenml/zen_server/dashboard/assets/{NumberBox-CrN0_kqI.js → NumberBox-T9eELfLZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{Partials-QLOZw624.js → Partials-YPBB3V4q.js} +1 -1
- zenml/zen_server/dashboard/assets/{PasswordChecker-DE71J_3F.js → PasswordChecker-CW0kqY0W.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProviderIcon-C16CCIN4.js → ProviderIcon-Bb3Xha5A.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProviderRadio-D3FuCHf3.js → ProviderRadio-BROY1700.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-BaMwHF-Q.js → SecretTooltip-C_qByGWB.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-DuIC65H9.js → SetPassword-7pRB00El.js} +1 -1
- zenml/zen_server/dashboard/assets/{Tick-DJTCF0Re.js → Tick-CykQFPj2.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CUm-DMpw.js → UpdatePasswordSchemas-DckMEkFf.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-CKw0juLF.js → UsageReason-DVceN14P.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-Cv9ApYWU.js → WizardFooter-CW0Cvd70.js} +1 -1
- zenml/zen_server/dashboard/assets/{check-circle-DOoS4yhF.js → check-circle-Dwxliy1Z.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-Cwb-W_B_.js → chevron-down-8wLBS5pQ.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-c9H46Kl8.js → chevron-right-double-DoD8iXWM.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BB4BVa6E.js → cloud-only-B-s_HMDm.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-BJYErIjr.js → code-browser-CZUQs3Wa.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaGlDsUy.js → copy-CaSMXwiU.js} +1 -1
- zenml/zen_server/dashboard/assets/{create-stack-F29xAUEx.js → create-stack-CEmaPZ4c.js} +1 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFAFXr2_.js → docker-BFNgg-z3.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-C6K59vUm.js → dots-horizontal-DK5Duzx4.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-BKXwSDK2.js → form-schemas-1AyOCx90.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-Dj6ntk0L.js → gcp-7M2Yf3ZK.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-CwN931fX.js → help-Dam461dC.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-5GJ5ysEZ.js → index-BVJ8n2-j.js} +1 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +1 -0
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +1 -0
- zenml/zen_server/dashboard/assets/{index-D-mtoBj3.js → index-DKPhqP2B.js} +1 -1
- zenml/zen_server/dashboard/assets/index-QQb7wQEC.js +55 -0
- zenml/zen_server/dashboard/assets/index.esm-cuVep_NJ.js +1 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-BLGc9T1a.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-ZzOtrSSW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-D4Vwzodn.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DH2hs3aW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BGBWG807.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-BuzJQFbW.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-BPnxn4xp.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-Dc4VJyIJ.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-BShVwWPj.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-CUiC4oBV.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-Bt9VVOA-.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-CaqZN2hq.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-BZzXV7-1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-DxZsaF_h.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-CeQXL5ds.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-d_eO-yCQ.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-CwicyhtI.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-Dhlb-90d.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-Bput3-QP.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-DCpCPQOf.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-B6guLgqG.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-M2hEX8vc.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-C9WLioJ8.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-Cnui8OiR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-BOOGhInR.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-gitzw0hO.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-D2bGa7uu.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-deR1Tlfd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-B5cFAncS.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-D273HNI0.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-C1t-h-pH.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-hnt3BR84.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-CIS2RHJS.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-UMdmhHu2.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-BnYJhD27.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-CAF0vJDd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-BUNmGMP1.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DMkecbls.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DOriooB6.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-DQPw2Hwd.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cc8MFFhd.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-Cm6aH8_k.woff +0 -0
- zenml/zen_server/dashboard/assets/{kubernetes-BjbR6D-1.js → kubernetes--g7r02Zu.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-Dru15_XR.js → layout-DCSYN7-C.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-BT2L8hAQ.js → link-external-CBEk6kEG.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-ax6iL2Mb.js → login-mutation-DTcAFP1l.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-GiDJXbLS.js → logs-D5bdJGur.js} +1 -1
- zenml/zen_server/dashboard/assets/not-found-Cc-JkRH2.js +1 -0
- zenml/zen_server/dashboard/assets/{package-DYKZ5jKW.js → package-Cs35Szwh.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-HkVBdZl6.js → page-9yplj5JT.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-uORspyRu.js → page-B-y2XKIc.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BDg1F-Ug.js → page-BQQKaabe.js} +2 -2
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CvKnNK1S.js → page-BjjlMk7s.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BukXK1Aa.js → page-Bvd7YH2A.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DcTjHmYZ.js → page-C0N5q3l7.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DuqYMYmH.js → page-C_f47pBf.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DRfcRK1w.js → page-ClUVkl-O.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +3 -0
- zenml/zen_server/dashboard/assets/{page-p0BhSAWx.js → page-CyN2bdWG.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +1 -0
- zenml/zen_server/dashboard/assets/{page-6mfzecin.js → page-DH_Z7iW1.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-8kYmrh0B.js → page-DIOXwhiD.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BXarY9K2.js → page-DLixvR-7.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-MAXyfXBq.js → page-Ddgy6kDS.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +9 -0
- zenml/zen_server/dashboard/assets/{page-CKHNAq7z.js → page-Dx16z7nA.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DMSLXKGT.js → page-DzpVUZ8f.js} +1 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-miU2rhYG.js → page-N4qoPHKb.js} +1 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +1 -0
- zenml/zen_server/dashboard/assets/{page-3qPX9WYH.js → page-bKaULTGG.js} +1 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-CfJMar_k.js → persist-CP0JmYZ4.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-BxIR2XZs.js → persist-DNb5cdrU.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-Bc8eLSDM.js → plus-C9IxgN2M.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-hfgWPeto.js → refresh-BVu22P_C.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-SESCGQXm.js → rocket-CONEmRmB.js} +1 -1
- zenml/zen_server/dashboard/assets/{sharedSchema-vub0rii3.js → sharedSchema-TMLu-nYQ.js} +1 -1
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-m-hJG8i9.js → tick-circle-CM1ZScbQ.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DUWZWzse.js → trash-DkJHMOg7.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-FGVP7X2U.js → update-server-settings-mutation-DsU8cNVl.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-CbAPzsmT.js → url-D5le3J4q.js} +1 -1
- zenml/zen_server/dashboard/assets/zod-D89GC_vc.js +1 -0
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/__init__.py +7 -16
- zenml/zen_server/deploy/base_provider.py +46 -73
- zenml/zen_server/deploy/{local → daemon}/__init__.py +3 -3
- zenml/zen_server/deploy/{local/local_provider.py → daemon/daemon_provider.py} +44 -63
- zenml/zen_server/deploy/{local/local_zen_server.py → daemon/daemon_zen_server.py} +50 -28
- zenml/zen_server/deploy/deployer.py +90 -171
- zenml/zen_server/deploy/deployment.py +20 -12
- zenml/zen_server/deploy/docker/docker_provider.py +9 -28
- zenml/zen_server/deploy/docker/docker_zen_server.py +19 -9
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +11 -0
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +1 -1
- zenml/zen_server/jwt.py +9 -9
- zenml/zen_server/routers/artifact_version_endpoints.py +1 -1
- zenml/zen_server/routers/auth_endpoints.py +30 -8
- zenml/zen_server/routers/runs_endpoints.py +0 -28
- zenml/zen_server/routers/stack_components_endpoints.py +1 -1
- zenml/zen_server/routers/workspaces_endpoints.py +1 -1
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +7 -4
- zenml/zen_server/template_execution/utils.py +8 -63
- zenml/zen_server/utils.py +67 -69
- zenml/zen_server/zen_server_api.py +107 -17
- zenml/zen_stores/base_zen_store.py +4 -51
- zenml/zen_stores/migrations/versions/0.67.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.68.1_release.py +23 -0
- zenml/zen_stores/migrations/versions/1d8f30c54477_migrate_to_new_.py +124 -0
- zenml/zen_stores/migrations/versions/c22561cbb3a9_add_artifact_unique_constraints.py +86 -0
- zenml/zen_stores/rest_zen_store.py +325 -147
- zenml/zen_stores/schemas/api_key_schemas.py +9 -4
- zenml/zen_stores/schemas/artifact_schemas.py +21 -2
- zenml/zen_stores/schemas/artifact_visualization_schemas.py +1 -1
- zenml/zen_stores/schemas/component_schemas.py +49 -6
- zenml/zen_stores/schemas/device_schemas.py +9 -4
- zenml/zen_stores/schemas/flavor_schemas.py +1 -1
- zenml/zen_stores/schemas/model_schemas.py +1 -1
- zenml/zen_stores/schemas/pipeline_run_schemas.py +12 -4
- zenml/zen_stores/schemas/service_schemas.py +1 -1
- zenml/zen_stores/schemas/step_run_schemas.py +3 -3
- zenml/zen_stores/schemas/trigger_schemas.py +1 -1
- zenml/zen_stores/sql_zen_store.py +422 -145
- zenml/zen_stores/template_utils.py +3 -1
- {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/METADATA +20 -18
- {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/RECORD +343 -505
- {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/WHEEL +1 -1
- zenml/api.py +0 -60
- zenml/cli/stack_recipes.py +0 -469
- zenml/lineage_graph/__init__.py +0 -34
- zenml/lineage_graph/lineage_graph.py +0 -244
- zenml/lineage_graph/node/__init__.py +0 -32
- zenml/lineage_graph/node/artifact_node.py +0 -51
- zenml/lineage_graph/node/step_node.py +0 -41
- zenml/model/model_version.py +0 -38
- zenml/new/pipelines/code_archive.py +0 -157
- zenml/new/pipelines/pipeline_decorator.py +0 -118
- zenml/new/steps/step_decorator.py +0 -176
- zenml/pipelines/base_pipeline.py +0 -273
- zenml/post_execution/__init__.py +0 -30
- zenml/post_execution/pipeline.py +0 -55
- zenml/post_execution/pipeline_run.py +0 -55
- zenml/services/terraform/__init__.py +0 -14
- zenml/services/terraform/terraform_service.py +0 -441
- zenml/steps/external_artifact.py +0 -26
- zenml/steps/step_environment.py +0 -108
- zenml/steps/step_output.py +0 -36
- zenml/utils/mlstacks_utils.py +0 -635
- zenml/utils/terraform_utils.py +0 -43
- zenml/zen_server/dashboard/assets/@radix-DnFH_oo1.js +0 -85
- zenml/zen_server/dashboard/assets/@react-router-APVeuk-U.js +0 -29
- zenml/zen_server/dashboard/assets/@reactflow-ytavUpwh.js +0 -17
- zenml/zen_server/dashboard/assets/@tanstack-QbMbTrh5.js +0 -22
- zenml/zen_server/dashboard/assets/ComponentBadge-gKR1OIwG.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-D6r6aK8J.js +0 -1
- zenml/zen_server/dashboard/assets/MarkdownVisualization-DS05sfBm.js +0 -14
- zenml/zen_server/dashboard/assets/SearchField-BzmfxS0L.js +0 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BA3R2Sey.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-CP0pcJ3U.js +0 -1
- zenml/zen_server/dashboard/assets/index-BhJ6ZJxv.css +0 -1
- zenml/zen_server/dashboard/assets/index-Ci0nJ8EZ.js +0 -55
- zenml/zen_server/dashboard/assets/index.esm-BE1uqCX5.js +0 -1
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-Df6ckaLK.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-400-normal-JrS_4yms.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-BRrLR67x.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-500-normal-DskEQOpE.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-DDpWG8g5.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-600-normal-iz1--dBq.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-CzG7Kr3z.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-400-normal-tyfMZHQw.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-DOnSzjnx.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-500-normal-Xebo2OyJ.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-DpA2xaRd.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-cyrillic-ext-600-normal-KAwcVx6H.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-DQXyrmoy.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-400-normal-DvIPHDQ7.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-BjpBGs91.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-500-normal-CmOavsDc.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-B-l8Lzzd.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-600-normal-BLZsI-P3.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-CIdlr5YK.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-400-normal-_Rr29XE2.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-2pdUafRD.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-500-normal-Dtavx3qw.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-BmtRFZgT.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-greek-ext-600-normal-CkBLDEl_.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-BT1H-PT_.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-400-normal-Cdi8t5Mu.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-D4I8BKCx.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-500-normal-kWhwEdDH.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-B2Ssfs8e.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-600-normal-Dbvh0wvx.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-8tIzm-yw.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-400-normal-D3W-OpO-.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-B9u8Q_zH.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-500-normal-SuUkSNTU.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-Dg0Bk0Yr.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-latin-ext-600-normal-ao35dkSb.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-Cnt0N5Vm.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-400-normal-DIOGfGLL.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-0i6yoQMg.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-500-normal-5IkPJ6Nk.woff2 +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-BQ_dbMbg.woff +0 -0
- zenml/zen_server/dashboard/assets/inter-vietnamese-600-normal-DjnxGF-L.woff2 +0 -0
- zenml/zen_server/dashboard/assets/not-found-DbjllLY_.js +0 -1
- zenml/zen_server/dashboard/assets/page-B1n7_W7z.js +0 -1
- zenml/zen_server/dashboard/assets/page-BZZhLo2u.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bbf_oBjn.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjuBvZG.js +0 -9
- zenml/zen_server/dashboard/assets/page-CHaQkFK5.js +0 -1
- zenml/zen_server/dashboard/assets/page-CS0SYFK8.js +0 -1
- zenml/zen_server/dashboard/assets/page-DGM1CbYT.js +0 -2
- zenml/zen_server/dashboard/assets/page-DOmIZ2ra.js +0 -1
- zenml/zen_server/dashboard/assets/page-DYVmJ9_w.js +0 -3
- zenml/zen_server/dashboard/assets/page-Dwow2doB.js +0 -1
- zenml/zen_server/dashboard/assets/stack-detail-query-DQcyzG-2.js +0 -1
- zenml/zen_server/dashboard/assets/zod-uFd1wBcd.js +0 -1
- zenml/zen_server/dashboard_legacy/_redirects +0 -1
- zenml/zen_server/dashboard_legacy/asset-manifest.json +0 -131
- zenml/zen_server/dashboard_legacy/favicon.ico +0 -0
- zenml/zen_server/dashboard_legacy/index.html +0 -1
- zenml/zen_server/dashboard_legacy/manifest.json +0 -25
- zenml/zen_server/dashboard_legacy/precache-manifest.6d320abb70db612019dda6c4948e7a90.js +0 -462
- zenml/zen_server/dashboard_legacy/robots.txt +0 -2
- zenml/zen_server/dashboard_legacy/service-worker.js +0 -39
- zenml/zen_server/dashboard_legacy/static/css/2.5b37d44a.chunk.css +0 -16
- zenml/zen_server/dashboard_legacy/static/css/2.5b37d44a.chunk.css.map +0 -1
- zenml/zen_server/dashboard_legacy/static/css/main.e79dff04.chunk.css +0 -2
- zenml/zen_server/dashboard_legacy/static/css/main.e79dff04.chunk.css.map +0 -1
- zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js +0 -3
- zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js.LICENSE.txt +0 -95
- zenml/zen_server/dashboard_legacy/static/js/2.7934458d.chunk.js.map +0 -1
- zenml/zen_server/dashboard_legacy/static/js/main.fa9299d5.chunk.js +0 -2
- zenml/zen_server/dashboard_legacy/static/js/main.fa9299d5.chunk.js.map +0 -1
- zenml/zen_server/dashboard_legacy/static/js/runtime-main.bfca2edd.js +0 -2
- zenml/zen_server/dashboard_legacy/static/js/runtime-main.bfca2edd.js.map +0 -1
- zenml/zen_server/dashboard_legacy/static/media/AlertTriangle.28aee535.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/ArrowSquareOut.abfb9bc7.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/Back.86c23a22.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/BookOpen.5cb101ff.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/BoundingBox.1eb98717.svg +0 -10
- zenml/zen_server/dashboard_legacy/static/media/Burger.9b1c67d7.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Cached.2381fb8d.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/Calendar.356e11c7.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/ChartBarHorizontal.0247447b.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/ChartLine.0d79e18d.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/ChatDots.2e1c9211.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Check.dad6beb2.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/CheckCircleFilled.c19566d0.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Checkbox.af50e31e.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/ChevronDown.f860ce32.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/ChevronDownLight.6642d756.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/ChevronLeft.f6edfcdb.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/CircleCheck.f98fd6ca.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/Clock.ffc9de95.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Close.74e9efbc.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/CloseWithBorder.6960930a.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/CloseWithoutBorder.cd6f71df.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/CloudArrowUp.0aecb235.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Code.ef0f33b5.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Config.0be63f8a.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/Connector.9fd46ef1.svg +0 -10
- zenml/zen_server/dashboard_legacy/static/media/Copy.36e2112a.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/Dashboard.d05787e0.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Data.b1c3b5f8.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Delete.3c361b28.svg +0 -8
- zenml/zen_server/dashboard_legacy/static/media/Docs.7541d478.svg +0 -7
- zenml/zen_server/dashboard_legacy/static/media/Download.fba04d87.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/Edit.490eb294.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/EmptyRightArrow.23749d01.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Example.6396cd37.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/Extension.1394cd4a.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Eye.d9e4ee62.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/Failed.0213c1a0.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/FileText.1f15bacd.svg +0 -7
- zenml/zen_server/dashboard_legacy/static/media/Filter.ab6b9c0d.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Folders.12b29887.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/FunnelFill.6df4c143.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/GitCommit.7dd9c2aa.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/GitHub_Logo.cefc2023.png +0 -0
- zenml/zen_server/dashboard_legacy/static/media/Graph.2c63a892.svg +0 -11
- zenml/zen_server/dashboard_legacy/static/media/History.08329240.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Home.0843b0d5.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/ImageBuilder.ea762d9c.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/InProgress.304a0edc.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/Info.9fe10c5c.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/KeyboardReturn.491afbe3.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Link.72bbb55d.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/Lock.30f5e1fe.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/Lock2.a769ea52.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/LockKey.92f21621.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Logs.8bf4d005.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/MinusCircle.4188f418.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/ModelRegistry.f0de050a.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/MultiUser.a2ba7c67.svg +0 -10
- zenml/zen_server/dashboard_legacy/static/media/PaginationFirst.92628634.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/PaginationLast.00d3c732.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/PaginationNext.86158845.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/PaginationPrev.60c18a88.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Pen.f2d831d4.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/PhotoCamera.179d6d4c.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Pipeline.30d298b0.svg +0 -7
- zenml/zen_server/dashboard_legacy/static/media/Plus.5aa1c16b.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/PlusCircle.92d860dd.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/Repositories.71a36b8c.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/RightArrow.f30d3871.svg +0 -29
- zenml/zen_server/dashboard_legacy/static/media/Rocket.63bf7b9d.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/RocketLaunch.1bff2b59.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Rubik-Medium.c87313aa.ttf +0 -0
- zenml/zen_server/dashboard_legacy/static/media/Rubik-Regular.b3d0902b.ttf +0 -0
- zenml/zen_server/dashboard_legacy/static/media/Run.daec4fb2.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Search.d1afcce5.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/Settings.59ca73ae.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/Share2.46c3ff66.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/SignOut.6aa718c5.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/SimplePlus.5cf7ec20.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/SingleUser.bef3a095.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/SourceCodePro-Regular.b484b32f.ttf +0 -0
- zenml/zen_server/dashboard_legacy/static/media/Stack.19b604ac.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/StackComponent.b1ba90b5.svg +0 -4
- zenml/zen_server/dashboard_legacy/static/media/Star.f0c25022.svg +0 -9
- zenml/zen_server/dashboard_legacy/static/media/StarOutline.94ca8cd9.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Storefront.4b4796fe.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/Stream.543e3039.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/SupportAgent.510ddf1f.svg +0 -8
- zenml/zen_server/dashboard_legacy/static/media/Table.77033750.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Tool.d5785486.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/UserPlus.741a99d7.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/Verified.0625b2a0.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/addNew.4fb6c939.svg +0 -8
- zenml/zen_server/dashboard_legacy/static/media/arrowClose.cbd53f3f.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/arrowOpen.6ceef0af.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/check_small.30bc0138.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/circleArrowSideClose.98d6013e.svg +0 -18
- zenml/zen_server/dashboard_legacy/static/media/circleArrowSideOpen.63653df6.svg +0 -18
- zenml/zen_server/dashboard_legacy/static/media/image.104fd14b.png +0 -0
- zenml/zen_server/dashboard_legacy/static/media/imageAddIcon.e83004a9.svg +0 -7
- zenml/zen_server/dashboard_legacy/static/media/logo.93333e5c.svg +0 -1
- zenml/zen_server/dashboard_legacy/static/media/logo_small.4204397d.svg +0 -3
- zenml/zen_server/dashboard_legacy/static/media/logo_white.d4b4414e.svg +0 -20
- zenml/zen_server/dashboard_legacy/static/media/notConnected.5e2c8ea7.svg +0 -8
- zenml/zen_server/dashboard_legacy/static/media/plugin-fallback.72c294e6.svg +0 -6
- zenml/zen_server/dashboard_legacy/static/media/share.bcd998b0.svg +0 -5
- zenml/zen_server/dashboard_legacy/static/media/stars.08a9b19a.svg +0 -8
- zenml/zen_server/deploy/terraform/__init__.py +0 -41
- zenml/zen_server/deploy/terraform/providers/aws_provider.py +0 -61
- zenml/zen_server/deploy/terraform/providers/azure_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/gcp_provider.py +0 -59
- zenml/zen_server/deploy/terraform/providers/terraform_provider.py +0 -331
- zenml/zen_server/deploy/terraform/recipes/aws/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/aws/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/aws/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/aws/outputs.tf +0 -7
- zenml/zen_server/deploy/terraform/recipes/aws/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/aws/sql.tf +0 -62
- zenml/zen_server/deploy/terraform/recipes/aws/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/aws/variables.tf +0 -167
- zenml/zen_server/deploy/terraform/recipes/aws/vpc.tf +0 -47
- zenml/zen_server/deploy/terraform/recipes/aws/zen_server.tf +0 -103
- zenml/zen_server/deploy/terraform/recipes/azure/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/azure/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/azure/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/azure/key_vault.tf +0 -73
- zenml/zen_server/deploy/terraform/recipes/azure/outputs.tf +0 -7
- zenml/zen_server/deploy/terraform/recipes/azure/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/azure/rg.tf +0 -36
- zenml/zen_server/deploy/terraform/recipes/azure/sql.tf +0 -65
- zenml/zen_server/deploy/terraform/recipes/azure/terraform.tf +0 -52
- zenml/zen_server/deploy/terraform/recipes/azure/variables.tf +0 -176
- zenml/zen_server/deploy/terraform/recipes/azure/zen_server.tf +0 -103
- zenml/zen_server/deploy/terraform/recipes/gcp/.gitignore +0 -8
- zenml/zen_server/deploy/terraform/recipes/gcp/helm.tf +0 -20
- zenml/zen_server/deploy/terraform/recipes/gcp/ingress.tf +0 -30
- zenml/zen_server/deploy/terraform/recipes/gcp/outputs.tf +0 -7
- zenml/zen_server/deploy/terraform/recipes/gcp/printf.cmd +0 -2
- zenml/zen_server/deploy/terraform/recipes/gcp/sql.tf +0 -64
- zenml/zen_server/deploy/terraform/recipes/gcp/terraform.tf +0 -44
- zenml/zen_server/deploy/terraform/recipes/gcp/variables.tf +0 -171
- zenml/zen_server/deploy/terraform/recipes/gcp/zen_server.tf +0 -114
- zenml/zen_server/deploy/terraform/terraform_zen_server.py +0 -246
- /zenml/{new/pipelines → pipelines}/build_utils.py +0 -0
- /zenml/{new/steps → steps}/decorated_step.py +0 -0
- {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.66.0.dev20240928.dist-info → zenml_nightly-0.68.1.dev20241102.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,574 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Utilities for creating step runs."""
|
15
|
+
|
16
|
+
from datetime import datetime
|
17
|
+
from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
|
18
|
+
|
19
|
+
from zenml.client import Client
|
20
|
+
from zenml.config.step_configurations import ArtifactConfiguration, Step
|
21
|
+
from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
|
22
|
+
from zenml.enums import ExecutionStatus
|
23
|
+
from zenml.logger import get_logger
|
24
|
+
from zenml.model.utils import link_artifact_version_to_model_version
|
25
|
+
from zenml.models import (
|
26
|
+
ArtifactVersionResponse,
|
27
|
+
ModelVersionPipelineRunRequest,
|
28
|
+
ModelVersionResponse,
|
29
|
+
PipelineDeploymentResponse,
|
30
|
+
PipelineRunResponse,
|
31
|
+
PipelineRunUpdate,
|
32
|
+
StepRunRequest,
|
33
|
+
StepRunResponse,
|
34
|
+
StepRunUpdate,
|
35
|
+
)
|
36
|
+
from zenml.orchestrators import cache_utils, input_utils, utils
|
37
|
+
from zenml.stack import Stack
|
38
|
+
from zenml.utils import pagination_utils, string_utils
|
39
|
+
|
40
|
+
if TYPE_CHECKING:
|
41
|
+
from zenml.model.model import Model
|
42
|
+
|
43
|
+
logger = get_logger(__name__)
|
44
|
+
|
45
|
+
|
46
|
+
class StepRunRequestFactory:
|
47
|
+
"""Helper class to create step run requests."""
|
48
|
+
|
49
|
+
def __init__(
|
50
|
+
self,
|
51
|
+
deployment: "PipelineDeploymentResponse",
|
52
|
+
pipeline_run: "PipelineRunResponse",
|
53
|
+
stack: "Stack",
|
54
|
+
) -> None:
|
55
|
+
"""Initialize the object.
|
56
|
+
|
57
|
+
Args:
|
58
|
+
deployment: The deployment for which to create step run requests.
|
59
|
+
pipeline_run: The pipeline run for which to create step run
|
60
|
+
requests.
|
61
|
+
stack: The stack on which the pipeline run is happening.
|
62
|
+
"""
|
63
|
+
self.deployment = deployment
|
64
|
+
self.pipeline_run = pipeline_run
|
65
|
+
self.stack = stack
|
66
|
+
|
67
|
+
def create_request(self, invocation_id: str) -> StepRunRequest:
|
68
|
+
"""Create a step run request.
|
69
|
+
|
70
|
+
This will only create a request with basic information and will not yet
|
71
|
+
compute information like the cache key and inputs. This is separated
|
72
|
+
into a different method `populate_request(...)` that might raise
|
73
|
+
exceptions while trying to compute this information.
|
74
|
+
|
75
|
+
Args:
|
76
|
+
invocation_id: The invocation ID for which to create the request.
|
77
|
+
|
78
|
+
Returns:
|
79
|
+
The step run request.
|
80
|
+
"""
|
81
|
+
return StepRunRequest(
|
82
|
+
name=invocation_id,
|
83
|
+
pipeline_run_id=self.pipeline_run.id,
|
84
|
+
deployment=self.deployment.id,
|
85
|
+
status=ExecutionStatus.RUNNING,
|
86
|
+
start_time=datetime.utcnow(),
|
87
|
+
user=Client().active_user.id,
|
88
|
+
workspace=Client().active_workspace.id,
|
89
|
+
)
|
90
|
+
|
91
|
+
def populate_request(self, request: StepRunRequest) -> None:
|
92
|
+
"""Populate a step run request with additional information.
|
93
|
+
|
94
|
+
Args:
|
95
|
+
request: The request to populate.
|
96
|
+
"""
|
97
|
+
step = self.deployment.step_configurations[request.name]
|
98
|
+
|
99
|
+
input_artifacts, parent_step_ids = input_utils.resolve_step_inputs(
|
100
|
+
step=step,
|
101
|
+
pipeline_run=self.pipeline_run,
|
102
|
+
)
|
103
|
+
input_artifact_ids = {
|
104
|
+
input_name: artifact.id
|
105
|
+
for input_name, artifact in input_artifacts.items()
|
106
|
+
}
|
107
|
+
request.inputs = input_artifact_ids
|
108
|
+
request.parent_step_ids = parent_step_ids
|
109
|
+
|
110
|
+
cache_key = cache_utils.generate_cache_key(
|
111
|
+
step=step,
|
112
|
+
input_artifact_ids=input_artifact_ids,
|
113
|
+
artifact_store=self.stack.artifact_store,
|
114
|
+
workspace_id=Client().active_workspace.id,
|
115
|
+
)
|
116
|
+
request.cache_key = cache_key
|
117
|
+
|
118
|
+
(
|
119
|
+
docstring,
|
120
|
+
source_code,
|
121
|
+
) = self._get_docstring_and_source_code(invocation_id=request.name)
|
122
|
+
|
123
|
+
request.docstring = docstring
|
124
|
+
request.source_code = source_code
|
125
|
+
request.code_hash = step.config.parameters.get(
|
126
|
+
CODE_HASH_PARAMETER_NAME
|
127
|
+
)
|
128
|
+
|
129
|
+
cache_enabled = utils.is_setting_enabled(
|
130
|
+
is_enabled_on_step=step.config.enable_cache,
|
131
|
+
is_enabled_on_pipeline=self.deployment.pipeline_configuration.enable_cache,
|
132
|
+
)
|
133
|
+
|
134
|
+
if cache_enabled:
|
135
|
+
if cached_step_run := cache_utils.get_cached_step_run(
|
136
|
+
cache_key=cache_key
|
137
|
+
):
|
138
|
+
request.inputs = {
|
139
|
+
input_name: artifact.id
|
140
|
+
for input_name, artifact in cached_step_run.inputs.items()
|
141
|
+
}
|
142
|
+
|
143
|
+
request.original_step_run_id = cached_step_run.id
|
144
|
+
request.outputs = {
|
145
|
+
output_name: artifact.id
|
146
|
+
for output_name, artifact in cached_step_run.outputs.items()
|
147
|
+
}
|
148
|
+
|
149
|
+
request.status = ExecutionStatus.CACHED
|
150
|
+
request.end_time = request.start_time
|
151
|
+
|
152
|
+
def _get_docstring_and_source_code(
|
153
|
+
self, invocation_id: str
|
154
|
+
) -> Tuple[Optional[str], Optional[str]]:
|
155
|
+
"""Get the docstring and source code for the step.
|
156
|
+
|
157
|
+
Args:
|
158
|
+
invocation_id: The step invocation ID for which to get the
|
159
|
+
docstring and source code.
|
160
|
+
|
161
|
+
Returns:
|
162
|
+
The docstring and source code of the step.
|
163
|
+
"""
|
164
|
+
step = self.deployment.step_configurations[invocation_id]
|
165
|
+
|
166
|
+
try:
|
167
|
+
return self._get_docstring_and_source_code_from_step_instance(
|
168
|
+
step=step
|
169
|
+
)
|
170
|
+
except ImportError:
|
171
|
+
pass
|
172
|
+
|
173
|
+
# Failed to import the step instance, this is most likely because this
|
174
|
+
# code is running on the server as part of a template execution.
|
175
|
+
# We now try to fetch the docstring/source code from a step run of the
|
176
|
+
# deployment that was used to create the template
|
177
|
+
return self._try_to_get_docstring_and_source_code_from_template(
|
178
|
+
invocation_id=invocation_id
|
179
|
+
)
|
180
|
+
|
181
|
+
@staticmethod
|
182
|
+
def _get_docstring_and_source_code_from_step_instance(
|
183
|
+
step: "Step",
|
184
|
+
) -> Tuple[Optional[str], str]:
|
185
|
+
"""Get the docstring and source code of a step.
|
186
|
+
|
187
|
+
Args:
|
188
|
+
step: The step configuration.
|
189
|
+
|
190
|
+
Returns:
|
191
|
+
The docstring and source code of a step.
|
192
|
+
"""
|
193
|
+
from zenml.steps.base_step import BaseStep
|
194
|
+
|
195
|
+
step_instance = BaseStep.load_from_source(step.spec.source)
|
196
|
+
|
197
|
+
docstring = step_instance.docstring
|
198
|
+
if docstring and len(docstring) > TEXT_FIELD_MAX_LENGTH:
|
199
|
+
docstring = docstring[: (TEXT_FIELD_MAX_LENGTH - 3)] + "..."
|
200
|
+
|
201
|
+
source_code = step_instance.source_code
|
202
|
+
if source_code and len(source_code) > TEXT_FIELD_MAX_LENGTH:
|
203
|
+
source_code = source_code[: (TEXT_FIELD_MAX_LENGTH - 3)] + "..."
|
204
|
+
|
205
|
+
return docstring, source_code
|
206
|
+
|
207
|
+
def _try_to_get_docstring_and_source_code_from_template(
|
208
|
+
self, invocation_id: str
|
209
|
+
) -> Tuple[Optional[str], Optional[str]]:
|
210
|
+
"""Try to get the docstring and source code via a potential template.
|
211
|
+
|
212
|
+
Args:
|
213
|
+
invocation_id: The step invocation ID for which to get the
|
214
|
+
docstring and source code.
|
215
|
+
|
216
|
+
Returns:
|
217
|
+
The docstring and source code of the step.
|
218
|
+
"""
|
219
|
+
if template_id := self.pipeline_run.template_id:
|
220
|
+
template = Client().get_run_template(template_id)
|
221
|
+
if (
|
222
|
+
deployment_id := template.source_deployment.id
|
223
|
+
if template.source_deployment
|
224
|
+
else None
|
225
|
+
):
|
226
|
+
steps = Client().list_run_steps(
|
227
|
+
deployment_id=deployment_id,
|
228
|
+
name=invocation_id,
|
229
|
+
size=1,
|
230
|
+
hydrate=True,
|
231
|
+
)
|
232
|
+
|
233
|
+
if len(steps) > 0:
|
234
|
+
step = steps[0]
|
235
|
+
return step.docstring, step.source_code
|
236
|
+
|
237
|
+
return None, None
|
238
|
+
|
239
|
+
|
240
|
+
def find_cacheable_invocation_candidates(
|
241
|
+
deployment: "PipelineDeploymentResponse",
|
242
|
+
finished_invocations: Set[str],
|
243
|
+
) -> Set[str]:
|
244
|
+
"""Find invocations that can potentially be cached.
|
245
|
+
|
246
|
+
Args:
|
247
|
+
deployment: The pipeline deployment containing the invocations.
|
248
|
+
finished_invocations: A set of invocations that are already finished.
|
249
|
+
|
250
|
+
Returns:
|
251
|
+
The set of invocations that can potentially be cached.
|
252
|
+
"""
|
253
|
+
invocations = set()
|
254
|
+
for invocation_id, step in deployment.step_configurations.items():
|
255
|
+
if invocation_id in finished_invocations:
|
256
|
+
continue
|
257
|
+
|
258
|
+
cache_enabled = utils.is_setting_enabled(
|
259
|
+
is_enabled_on_step=step.config.enable_cache,
|
260
|
+
is_enabled_on_pipeline=deployment.pipeline_configuration.enable_cache,
|
261
|
+
)
|
262
|
+
|
263
|
+
if not cache_enabled:
|
264
|
+
continue
|
265
|
+
|
266
|
+
if set(step.spec.upstream_steps) - finished_invocations:
|
267
|
+
continue
|
268
|
+
|
269
|
+
invocations.add(invocation_id)
|
270
|
+
|
271
|
+
return invocations
|
272
|
+
|
273
|
+
|
274
|
+
def create_cached_step_runs(
|
275
|
+
deployment: "PipelineDeploymentResponse",
|
276
|
+
pipeline_run: PipelineRunResponse,
|
277
|
+
stack: "Stack",
|
278
|
+
) -> Set[str]:
|
279
|
+
"""Create all cached step runs for a pipeline run.
|
280
|
+
|
281
|
+
Args:
|
282
|
+
deployment: The deployment of the pipeline run.
|
283
|
+
pipeline_run: The pipeline run for which to create the step runs.
|
284
|
+
stack: The stack on which the pipeline run is happening.
|
285
|
+
|
286
|
+
Returns:
|
287
|
+
The invocation IDs of the created step runs.
|
288
|
+
"""
|
289
|
+
cached_invocations: Set[str] = set()
|
290
|
+
visited_invocations: Set[str] = set()
|
291
|
+
request_factory = StepRunRequestFactory(
|
292
|
+
deployment=deployment, pipeline_run=pipeline_run, stack=stack
|
293
|
+
)
|
294
|
+
|
295
|
+
pipeline_model_version, pipeline_run = prepare_pipeline_run_model_version(
|
296
|
+
pipeline_run=pipeline_run
|
297
|
+
)
|
298
|
+
|
299
|
+
while (
|
300
|
+
cache_candidates := find_cacheable_invocation_candidates(
|
301
|
+
deployment=deployment,
|
302
|
+
finished_invocations=cached_invocations,
|
303
|
+
)
|
304
|
+
# We've already checked these invocations and were not able to cache
|
305
|
+
# them -> no need to check them again
|
306
|
+
- visited_invocations
|
307
|
+
):
|
308
|
+
for invocation_id in cache_candidates:
|
309
|
+
visited_invocations.add(invocation_id)
|
310
|
+
|
311
|
+
try:
|
312
|
+
step_run_request = request_factory.create_request(
|
313
|
+
invocation_id
|
314
|
+
)
|
315
|
+
request_factory.populate_request(step_run_request)
|
316
|
+
except Exception as e:
|
317
|
+
# We failed to create/populate the step run. This might be due
|
318
|
+
# to some input resolution error, or an error importing the step
|
319
|
+
# source (there might be some missing dependencies). We just
|
320
|
+
# treat the step as not cached and let the orchestrator try
|
321
|
+
# again in a potentially different environment.
|
322
|
+
logger.debug(
|
323
|
+
"Failed preparing step run `%s`: %s", invocation_id, str(e)
|
324
|
+
)
|
325
|
+
continue
|
326
|
+
|
327
|
+
if step_run_request.status != ExecutionStatus.CACHED:
|
328
|
+
# If we're not able to cache the step run, the orchestrator
|
329
|
+
# will run the step later which will create the step run
|
330
|
+
# -> We don't need to do anything here
|
331
|
+
continue
|
332
|
+
|
333
|
+
step_run = Client().zen_store.create_run_step(step_run_request)
|
334
|
+
|
335
|
+
# Refresh the pipeline run here to make sure we have the latest
|
336
|
+
# state
|
337
|
+
pipeline_run = Client().get_pipeline_run(pipeline_run.id)
|
338
|
+
|
339
|
+
step_model_version, step_run = prepare_step_run_model_version(
|
340
|
+
step_run=step_run, pipeline_run=pipeline_run
|
341
|
+
)
|
342
|
+
|
343
|
+
if model_version := step_model_version or pipeline_model_version:
|
344
|
+
link_output_artifacts_to_model_version(
|
345
|
+
artifacts=step_run.outputs,
|
346
|
+
output_configurations=step_run.config.outputs,
|
347
|
+
model_version=model_version,
|
348
|
+
)
|
349
|
+
|
350
|
+
logger.info("Using cached version of step `%s`.", invocation_id)
|
351
|
+
cached_invocations.add(invocation_id)
|
352
|
+
|
353
|
+
return cached_invocations
|
354
|
+
|
355
|
+
|
356
|
+
def get_or_create_model_version_for_pipeline_run(
|
357
|
+
model: "Model", pipeline_run: PipelineRunResponse
|
358
|
+
) -> Tuple[ModelVersionResponse, bool]:
|
359
|
+
"""Get or create a model version as part of a pipeline run.
|
360
|
+
|
361
|
+
Args:
|
362
|
+
model: The model to get or create.
|
363
|
+
pipeline_run: The pipeline run for which the model should be created.
|
364
|
+
|
365
|
+
Returns:
|
366
|
+
The model version and a boolean indicating whether it was newly created
|
367
|
+
or not.
|
368
|
+
"""
|
369
|
+
# Copy the model before modifying it so we don't accidently modify
|
370
|
+
# configurations in which the model object is potentially referenced
|
371
|
+
model = model.model_copy()
|
372
|
+
|
373
|
+
if model.model_version_id:
|
374
|
+
return model._get_model_version(), False
|
375
|
+
elif model.version:
|
376
|
+
if isinstance(model.version, str):
|
377
|
+
start_time = pipeline_run.start_time or datetime.utcnow()
|
378
|
+
model.version = string_utils.format_name_template(
|
379
|
+
model.version,
|
380
|
+
date=start_time.strftime("%Y_%m_%d"),
|
381
|
+
time=start_time.strftime("%H_%M_%S_%f"),
|
382
|
+
)
|
383
|
+
|
384
|
+
return (
|
385
|
+
model._get_or_create_model_version(),
|
386
|
+
model._created_model_version,
|
387
|
+
)
|
388
|
+
|
389
|
+
# The model version should be created as part of this run
|
390
|
+
# -> We first check if it was already created as part of this run, and if
|
391
|
+
# not we do create it. If this is running in two parallel steps, we might
|
392
|
+
# run into issues that this will create two versions. Ideally, all model
|
393
|
+
# versions required for a pipeline run and its steps could be created
|
394
|
+
# server-side at run creation time before the first step starts.
|
395
|
+
if model_version := get_model_version_created_by_pipeline_run(
|
396
|
+
model_name=model.name, pipeline_run=pipeline_run
|
397
|
+
):
|
398
|
+
return model_version, False
|
399
|
+
else:
|
400
|
+
return model._get_or_create_model_version(), True
|
401
|
+
|
402
|
+
|
403
|
+
def get_model_version_created_by_pipeline_run(
|
404
|
+
model_name: str, pipeline_run: PipelineRunResponse
|
405
|
+
) -> Optional[ModelVersionResponse]:
|
406
|
+
"""Get a model version that was created by a specific pipeline run.
|
407
|
+
|
408
|
+
This function does not refresh the pipeline run, so it will only try to
|
409
|
+
fetch the model version from existing steps if they're already part of the
|
410
|
+
response.
|
411
|
+
|
412
|
+
Args:
|
413
|
+
model_name: The model name for which to get the version.
|
414
|
+
pipeline_run: The pipeline run for which to get the version.
|
415
|
+
|
416
|
+
Returns:
|
417
|
+
A model version with the given name created by the run, or None if such
|
418
|
+
a model version does not exist.
|
419
|
+
"""
|
420
|
+
if pipeline_run.config.model and pipeline_run.model_version:
|
421
|
+
if (
|
422
|
+
pipeline_run.config.model.name == model_name
|
423
|
+
and pipeline_run.config.model.version is None
|
424
|
+
):
|
425
|
+
return pipeline_run.model_version
|
426
|
+
|
427
|
+
# We fetch a list of hydrated step runs here in order to avoid hydration
|
428
|
+
# calls for each step separately.
|
429
|
+
candidate_step_runs = pagination_utils.depaginate(
|
430
|
+
Client().list_run_steps,
|
431
|
+
pipeline_run_id=pipeline_run.id,
|
432
|
+
model=model_name,
|
433
|
+
hydrate=True,
|
434
|
+
)
|
435
|
+
for step_run in candidate_step_runs:
|
436
|
+
if step_run.config.model and step_run.model_version:
|
437
|
+
if (
|
438
|
+
step_run.config.model.name == model_name
|
439
|
+
and step_run.config.model.version is None
|
440
|
+
):
|
441
|
+
return step_run.model_version
|
442
|
+
|
443
|
+
return None
|
444
|
+
|
445
|
+
|
446
|
+
def prepare_pipeline_run_model_version(
|
447
|
+
pipeline_run: PipelineRunResponse,
|
448
|
+
) -> Tuple[Optional[ModelVersionResponse], PipelineRunResponse]:
|
449
|
+
"""Prepare the model version for a pipeline run.
|
450
|
+
|
451
|
+
Args:
|
452
|
+
pipeline_run: The pipeline run for which to prepare the model version.
|
453
|
+
|
454
|
+
Returns:
|
455
|
+
The prepared model version and the updated pipeline run.
|
456
|
+
"""
|
457
|
+
model_version = None
|
458
|
+
|
459
|
+
if pipeline_run.model_version:
|
460
|
+
model_version = pipeline_run.model_version
|
461
|
+
elif config_model := pipeline_run.config.model:
|
462
|
+
model_version, _ = get_or_create_model_version_for_pipeline_run(
|
463
|
+
model=config_model, pipeline_run=pipeline_run
|
464
|
+
)
|
465
|
+
pipeline_run = Client().zen_store.update_run(
|
466
|
+
run_id=pipeline_run.id,
|
467
|
+
run_update=PipelineRunUpdate(model_version_id=model_version.id),
|
468
|
+
)
|
469
|
+
link_pipeline_run_to_model_version(
|
470
|
+
pipeline_run=pipeline_run, model_version=model_version
|
471
|
+
)
|
472
|
+
log_model_version_dashboard_url(model_version)
|
473
|
+
|
474
|
+
return model_version, pipeline_run
|
475
|
+
|
476
|
+
|
477
|
+
def prepare_step_run_model_version(
|
478
|
+
step_run: StepRunResponse, pipeline_run: PipelineRunResponse
|
479
|
+
) -> Tuple[Optional[ModelVersionResponse], StepRunResponse]:
|
480
|
+
"""Prepare the model version for a step run.
|
481
|
+
|
482
|
+
Args:
|
483
|
+
step_run: The step run for which to prepare the model version.
|
484
|
+
pipeline_run: The pipeline run of the step.
|
485
|
+
|
486
|
+
Returns:
|
487
|
+
The prepared model version and the updated step run.
|
488
|
+
"""
|
489
|
+
model_version = None
|
490
|
+
|
491
|
+
if step_run.model_version:
|
492
|
+
model_version = step_run.model_version
|
493
|
+
elif config_model := step_run.config.model:
|
494
|
+
model_version, created = get_or_create_model_version_for_pipeline_run(
|
495
|
+
model=config_model, pipeline_run=pipeline_run
|
496
|
+
)
|
497
|
+
step_run = Client().zen_store.update_run_step(
|
498
|
+
step_run_id=step_run.id,
|
499
|
+
step_run_update=StepRunUpdate(model_version_id=model_version.id),
|
500
|
+
)
|
501
|
+
link_pipeline_run_to_model_version(
|
502
|
+
pipeline_run=pipeline_run, model_version=model_version
|
503
|
+
)
|
504
|
+
if created:
|
505
|
+
log_model_version_dashboard_url(model_version)
|
506
|
+
|
507
|
+
return model_version, step_run
|
508
|
+
|
509
|
+
|
510
|
+
def log_model_version_dashboard_url(
|
511
|
+
model_version: ModelVersionResponse,
|
512
|
+
) -> None:
|
513
|
+
"""Log the dashboard URL for a model version.
|
514
|
+
|
515
|
+
If the current server is not a ZenML Pro tenant, a fallback message is
|
516
|
+
logged instead.
|
517
|
+
|
518
|
+
Args:
|
519
|
+
model_version: The model version for which to log the dashboard URL.
|
520
|
+
"""
|
521
|
+
from zenml.utils.cloud_utils import try_get_model_version_url
|
522
|
+
|
523
|
+
if model_version_url_logs := try_get_model_version_url(model_version):
|
524
|
+
logger.info(model_version_url_logs)
|
525
|
+
else:
|
526
|
+
logger.info(
|
527
|
+
"Models can be viewed in the dashboard using ZenML Pro. Sign up "
|
528
|
+
"for a free trial at https://www.zenml.io/pro/"
|
529
|
+
)
|
530
|
+
|
531
|
+
|
532
|
+
def link_pipeline_run_to_model_version(
|
533
|
+
pipeline_run: PipelineRunResponse, model_version: ModelVersionResponse
|
534
|
+
) -> None:
|
535
|
+
"""Link a pipeline run to a model version.
|
536
|
+
|
537
|
+
Args:
|
538
|
+
pipeline_run: The pipeline run to link.
|
539
|
+
model_version: The model version to link.
|
540
|
+
"""
|
541
|
+
client = Client()
|
542
|
+
client.zen_store.create_model_version_pipeline_run_link(
|
543
|
+
ModelVersionPipelineRunRequest(
|
544
|
+
user=client.active_user.id,
|
545
|
+
workspace=client.active_workspace.id,
|
546
|
+
pipeline_run=pipeline_run.id,
|
547
|
+
model=model_version.model.id,
|
548
|
+
model_version=model_version.id,
|
549
|
+
)
|
550
|
+
)
|
551
|
+
|
552
|
+
|
553
|
+
def link_output_artifacts_to_model_version(
|
554
|
+
artifacts: Dict[str, ArtifactVersionResponse],
|
555
|
+
output_configurations: Mapping[str, ArtifactConfiguration],
|
556
|
+
model_version: ModelVersionResponse,
|
557
|
+
) -> None:
|
558
|
+
"""Link the outputs of a step run to a model version.
|
559
|
+
|
560
|
+
Args:
|
561
|
+
artifacts: The step output artifacts.
|
562
|
+
output_configurations: The output configurations for the step.
|
563
|
+
model_version: The model version to link.
|
564
|
+
"""
|
565
|
+
for output_name, output_artifact in artifacts.items():
|
566
|
+
artifact_config = None
|
567
|
+
if output_config := output_configurations.get(output_name, None):
|
568
|
+
artifact_config = output_config.artifact_config
|
569
|
+
|
570
|
+
link_artifact_version_to_model_version(
|
571
|
+
artifact_version=output_artifact,
|
572
|
+
model_version=model_version,
|
573
|
+
artifact_config=artifact_config,
|
574
|
+
)
|