zenml-nightly 0.68.1.dev20241103__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +77 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/artifacts/external_artifact.py +2 -1
- zenml/artifacts/{load_directory_materializer.py → preexisting_data_materializer.py} +8 -9
- zenml/artifacts/utils.py +139 -80
- zenml/cli/base.py +4 -4
- zenml/cli/model.py +1 -6
- zenml/cli/stack.py +1 -0
- zenml/client.py +29 -74
- zenml/config/server_config.py +17 -1
- zenml/constants.py +2 -7
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/enums.py +20 -4
- zenml/exceptions.py +4 -0
- zenml/integrations/__init__.py +3 -1
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
- zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
- zenml/integrations/constants.py +1 -0
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
- zenml/integrations/evidently/__init__.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +8 -12
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +2 -5
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -3
- zenml/integrations/langchain/__init__.py +2 -1
- zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
- zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
- zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
- zenml/integrations/openai/__init__.py +1 -1
- zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
- zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
- zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
- zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
- zenml/integrations/tensorflow/materializers/keras_materializer.py +11 -22
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +8 -15
- zenml/integrations/vllm/__init__.py +50 -0
- zenml/integrations/vllm/flavors/__init__.py +21 -0
- zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
- zenml/integrations/vllm/model_deployers/__init__.py +19 -0
- zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
- zenml/integrations/vllm/services/__init__.py +19 -0
- zenml/integrations/vllm/services/vllm_deployment.py +206 -0
- zenml/integrations/whylogs/materializers/whylogs_materializer.py +11 -18
- zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
- zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
- zenml/materializers/base_materializer.py +68 -1
- zenml/metadata/lazy_load.py +20 -7
- zenml/model/model.py +17 -64
- zenml/model/utils.py +5 -0
- zenml/models/__init__.py +0 -12
- zenml/models/v2/base/filter.py +121 -8
- zenml/models/v2/core/artifact_version.py +42 -7
- zenml/models/v2/core/model_version.py +26 -5
- zenml/models/v2/core/pipeline_run.py +25 -6
- zenml/models/v2/core/run_metadata.py +2 -217
- zenml/models/v2/core/step_run.py +62 -24
- zenml/orchestrators/base_orchestrator.py +12 -1
- zenml/orchestrators/input_utils.py +44 -19
- zenml/orchestrators/step_launcher.py +4 -3
- zenml/orchestrators/step_run_utils.py +19 -15
- zenml/orchestrators/step_runner.py +25 -14
- zenml/orchestrators/utils.py +45 -26
- zenml/stack/flavor.py +9 -5
- zenml/stack_deployments/aws_stack_deployment.py +23 -6
- zenml/stack_deployments/azure_stack_deployment.py +28 -5
- zenml/stack_deployments/gcp_stack_deployment.py +25 -8
- zenml/stack_deployments/stack_deployment.py +3 -5
- zenml/steps/base_step.py +1 -1
- zenml/steps/entrypoint_function_utils.py +3 -5
- zenml/steps/step_context.py +5 -2
- zenml/steps/utils.py +13 -2
- zenml/utils/callback_registry.py +71 -0
- zenml/utils/metadata_utils.py +335 -0
- zenml/zen_server/auth.py +221 -3
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
- zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
- zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
- zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
- zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
- zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
- zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
- zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
- zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
- zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
- zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
- zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
- zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
- zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
- zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
- zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
- zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
- zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
- zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
- zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
- zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
- zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
- zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
- zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +2 -0
- zenml/zen_server/jwt.py +30 -13
- zenml/zen_server/rbac/endpoint_utils.py +43 -1
- zenml/zen_server/rbac/utils.py +0 -2
- zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
- zenml/zen_server/routers/auth_endpoints.py +134 -102
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/workspaces_endpoints.py +3 -4
- zenml/zen_server/template_execution/utils.py +14 -16
- zenml/zen_server/utils.py +27 -0
- zenml/zen_server/zen_server_api.py +6 -3
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/rest_zen_store.py +68 -64
- zenml/zen_stores/schemas/artifact_schemas.py +8 -1
- zenml/zen_stores/schemas/model_schemas.py +27 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
- zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
- zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
- zenml/zen_stores/schemas/step_run_schemas.py +18 -10
- zenml/zen_stores/sql_zen_store.py +283 -219
- zenml/zen_stores/zen_store_interface.py +15 -42
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +226 -203
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
- zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
- zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
- zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
- zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
- zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
- zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
- zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
- zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
- zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
@@ -18,17 +18,19 @@ from uuid import UUID
|
|
18
18
|
|
19
19
|
from zenml.client import Client
|
20
20
|
from zenml.config.step_configurations import Step
|
21
|
+
from zenml.enums import ArtifactSaveType, StepRunInputArtifactType
|
21
22
|
from zenml.exceptions import InputResolutionError
|
22
23
|
from zenml.utils import pagination_utils
|
23
24
|
|
24
25
|
if TYPE_CHECKING:
|
25
|
-
from zenml.models import
|
26
|
+
from zenml.models import PipelineRunResponse
|
27
|
+
from zenml.models.v2.core.step_run import StepRunInputResponse
|
26
28
|
|
27
29
|
|
28
30
|
def resolve_step_inputs(
|
29
31
|
step: "Step",
|
30
32
|
pipeline_run: "PipelineRunResponse",
|
31
|
-
) -> Tuple[Dict[str, "
|
33
|
+
) -> Tuple[Dict[str, "StepRunInputResponse"], List[UUID]]:
|
32
34
|
"""Resolves inputs for the current step.
|
33
35
|
|
34
36
|
Args:
|
@@ -45,7 +47,8 @@ def resolve_step_inputs(
|
|
45
47
|
The IDs of the input artifact versions and the IDs of parent steps of
|
46
48
|
the current step.
|
47
49
|
"""
|
48
|
-
from zenml.models import ArtifactVersionResponse
|
50
|
+
from zenml.models import ArtifactVersionResponse
|
51
|
+
from zenml.models.v2.core.step_run import StepRunInputResponse
|
49
52
|
|
50
53
|
current_run_steps = {
|
51
54
|
run_step.name: run_step
|
@@ -54,7 +57,7 @@ def resolve_step_inputs(
|
|
54
57
|
)
|
55
58
|
}
|
56
59
|
|
57
|
-
input_artifacts: Dict[str,
|
60
|
+
input_artifacts: Dict[str, StepRunInputResponse] = {}
|
58
61
|
for name, input_ in step.spec.inputs.items():
|
59
62
|
try:
|
60
63
|
step_run = current_run_steps[input_.step_name]
|
@@ -64,22 +67,44 @@ def resolve_step_inputs(
|
|
64
67
|
)
|
65
68
|
|
66
69
|
try:
|
67
|
-
|
70
|
+
outputs = step_run.outputs[input_.output_name]
|
68
71
|
except KeyError:
|
69
72
|
raise InputResolutionError(
|
70
|
-
f"No output `{input_.output_name}` found for step "
|
73
|
+
f"No step output `{input_.output_name}` found for step "
|
71
74
|
f"`{input_.step_name}`."
|
72
75
|
)
|
73
76
|
|
74
|
-
|
77
|
+
step_outputs = [
|
78
|
+
output
|
79
|
+
for output in outputs
|
80
|
+
if output.save_type == ArtifactSaveType.STEP_OUTPUT
|
81
|
+
]
|
82
|
+
if len(step_outputs) > 2:
|
83
|
+
# This should never happen, there can only be a single regular step
|
84
|
+
# output for a name
|
85
|
+
raise InputResolutionError(
|
86
|
+
f"Too many step outputs for output `{input_.output_name}` of "
|
87
|
+
f"step `{input_.step_name}`."
|
88
|
+
)
|
89
|
+
elif len(step_outputs) == 0:
|
90
|
+
raise InputResolutionError(
|
91
|
+
f"No step output `{input_.output_name}` found for step "
|
92
|
+
f"`{input_.step_name}`."
|
93
|
+
)
|
94
|
+
|
95
|
+
input_artifacts[name] = StepRunInputResponse(
|
96
|
+
input_type=StepRunInputArtifactType.STEP_OUTPUT,
|
97
|
+
**step_outputs[0].model_dump(),
|
98
|
+
)
|
75
99
|
|
76
100
|
for (
|
77
101
|
name,
|
78
102
|
external_artifact,
|
79
103
|
) in step.config.external_input_artifacts.items():
|
80
104
|
artifact_version_id = external_artifact.get_artifact_version_id()
|
81
|
-
input_artifacts[name] =
|
82
|
-
|
105
|
+
input_artifacts[name] = StepRunInputResponse(
|
106
|
+
input_type=StepRunInputArtifactType.EXTERNAL,
|
107
|
+
**Client().get_artifact_version(artifact_version_id).model_dump(),
|
83
108
|
)
|
84
109
|
|
85
110
|
for name, config_ in step.config.model_artifacts_or_metadata.items():
|
@@ -98,9 +123,7 @@ def resolve_step_inputs(
|
|
98
123
|
):
|
99
124
|
# metadata values should go directly in parameters, as primitive types
|
100
125
|
step.config.parameters[name] = (
|
101
|
-
context_model_version.run_metadata[
|
102
|
-
config_.metadata_name
|
103
|
-
].value
|
126
|
+
context_model_version.run_metadata[config_.metadata_name]
|
104
127
|
)
|
105
128
|
elif config_.artifact_name is None:
|
106
129
|
err_msg = (
|
@@ -112,14 +135,15 @@ def resolve_step_inputs(
|
|
112
135
|
config_.artifact_name, config_.artifact_version
|
113
136
|
):
|
114
137
|
if config_.metadata_name is None:
|
115
|
-
input_artifacts[name] =
|
138
|
+
input_artifacts[name] = StepRunInputResponse(
|
139
|
+
input_type=StepRunInputArtifactType.LAZY_LOADED,
|
140
|
+
**artifact_.model_dump(),
|
141
|
+
)
|
116
142
|
elif config_.metadata_name:
|
117
143
|
# metadata values should go directly in parameters, as primitive types
|
118
144
|
try:
|
119
145
|
step.config.parameters[name] = (
|
120
|
-
artifact_.run_metadata[
|
121
|
-
config_.metadata_name
|
122
|
-
].value
|
146
|
+
artifact_.run_metadata[config_.metadata_name]
|
123
147
|
)
|
124
148
|
except KeyError:
|
125
149
|
err_msg = (
|
@@ -141,9 +165,10 @@ def resolve_step_inputs(
|
|
141
165
|
for name, cll_ in step.config.client_lazy_loaders.items():
|
142
166
|
value_ = cll_.evaluate()
|
143
167
|
if isinstance(value_, ArtifactVersionResponse):
|
144
|
-
input_artifacts[name] =
|
145
|
-
|
146
|
-
|
168
|
+
input_artifacts[name] = StepRunInputResponse(
|
169
|
+
input_type=StepRunInputArtifactType.LAZY_LOADED,
|
170
|
+
**value_.model_dump(),
|
171
|
+
)
|
147
172
|
else:
|
148
173
|
step.config.parameters[name] = value_
|
149
174
|
|
@@ -33,13 +33,13 @@ from zenml.environment import get_run_environment_dict
|
|
33
33
|
from zenml.logger import get_logger
|
34
34
|
from zenml.logging import step_logging
|
35
35
|
from zenml.models import (
|
36
|
-
ArtifactVersionResponse,
|
37
36
|
LogsRequest,
|
38
37
|
PipelineDeploymentResponse,
|
39
38
|
PipelineRunRequest,
|
40
39
|
PipelineRunResponse,
|
41
40
|
StepRunResponse,
|
42
41
|
)
|
42
|
+
from zenml.models.v2.core.step_run import StepRunInputResponse
|
43
43
|
from zenml.orchestrators import output_utils, publish_utils, step_run_utils
|
44
44
|
from zenml.orchestrators import utils as orchestrator_utils
|
45
45
|
from zenml.orchestrators.step_runner import StepRunner
|
@@ -422,7 +422,8 @@ class StepLauncher:
|
|
422
422
|
)
|
423
423
|
)
|
424
424
|
environment = orchestrator_utils.get_config_environment_vars(
|
425
|
-
|
425
|
+
pipeline_run_id=step_run_info.run_id,
|
426
|
+
step_run_id=step_run_info.step_run_id,
|
426
427
|
)
|
427
428
|
if last_retry:
|
428
429
|
environment[ENV_ZENML_IGNORE_FAILURE_HOOK] = str(False)
|
@@ -442,7 +443,7 @@ class StepLauncher:
|
|
442
443
|
pipeline_run: PipelineRunResponse,
|
443
444
|
step_run: StepRunResponse,
|
444
445
|
step_run_info: StepRunInfo,
|
445
|
-
input_artifacts: Dict[str,
|
446
|
+
input_artifacts: Dict[str, StepRunInputResponse],
|
446
447
|
output_artifact_uris: Dict[str, str],
|
447
448
|
last_retry: bool,
|
448
449
|
) -> None:
|
@@ -14,12 +14,12 @@
|
|
14
14
|
"""Utilities for creating step runs."""
|
15
15
|
|
16
16
|
from datetime import datetime
|
17
|
-
from typing import TYPE_CHECKING, Dict, Mapping, Optional, Set, Tuple
|
17
|
+
from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple
|
18
18
|
|
19
19
|
from zenml.client import Client
|
20
20
|
from zenml.config.step_configurations import ArtifactConfiguration, Step
|
21
21
|
from zenml.constants import CODE_HASH_PARAMETER_NAME, TEXT_FIELD_MAX_LENGTH
|
22
|
-
from zenml.enums import ExecutionStatus
|
22
|
+
from zenml.enums import ArtifactSaveType, ExecutionStatus
|
23
23
|
from zenml.logger import get_logger
|
24
24
|
from zenml.model.utils import link_artifact_version_to_model_version
|
25
25
|
from zenml.models import (
|
@@ -104,6 +104,7 @@ class StepRunRequestFactory:
|
|
104
104
|
input_name: artifact.id
|
105
105
|
for input_name, artifact in input_artifacts.items()
|
106
106
|
}
|
107
|
+
|
107
108
|
request.inputs = input_artifact_ids
|
108
109
|
request.parent_step_ids = parent_step_ids
|
109
110
|
|
@@ -142,8 +143,8 @@ class StepRunRequestFactory:
|
|
142
143
|
|
143
144
|
request.original_step_run_id = cached_step_run.id
|
144
145
|
request.outputs = {
|
145
|
-
output_name: artifact.id
|
146
|
-
for output_name,
|
146
|
+
output_name: [artifact.id for artifact in artifacts]
|
147
|
+
for output_name, artifacts in cached_step_run.outputs.items()
|
147
148
|
}
|
148
149
|
|
149
150
|
request.status = ExecutionStatus.CACHED
|
@@ -551,7 +552,7 @@ def link_pipeline_run_to_model_version(
|
|
551
552
|
|
552
553
|
|
553
554
|
def link_output_artifacts_to_model_version(
|
554
|
-
artifacts: Dict[str, ArtifactVersionResponse],
|
555
|
+
artifacts: Dict[str, List[ArtifactVersionResponse]],
|
555
556
|
output_configurations: Mapping[str, ArtifactConfiguration],
|
556
557
|
model_version: ModelVersionResponse,
|
557
558
|
) -> None:
|
@@ -562,13 +563,16 @@ def link_output_artifacts_to_model_version(
|
|
562
563
|
output_configurations: The output configurations for the step.
|
563
564
|
model_version: The model version to link.
|
564
565
|
"""
|
565
|
-
for output_name,
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
566
|
+
for output_name, output_artifacts in artifacts.items():
|
567
|
+
for output_artifact in output_artifacts:
|
568
|
+
artifact_config = None
|
569
|
+
if output_artifact.save_type == ArtifactSaveType.STEP_OUTPUT and (
|
570
|
+
output_config := output_configurations.get(output_name, None)
|
571
|
+
):
|
572
|
+
artifact_config = output_config.artifact_config
|
573
|
+
|
574
|
+
link_artifact_version_to_model_version(
|
575
|
+
artifact_version=output_artifact,
|
576
|
+
model_version=model_version,
|
577
|
+
artifact_config=artifact_config,
|
578
|
+
)
|
@@ -28,7 +28,8 @@ from typing import (
|
|
28
28
|
)
|
29
29
|
|
30
30
|
from zenml.artifacts.unmaterialized_artifact import UnmaterializedArtifact
|
31
|
-
from zenml.artifacts.utils import
|
31
|
+
from zenml.artifacts.utils import _store_artifact_data_and_prepare_request
|
32
|
+
from zenml.client import Client
|
32
33
|
from zenml.config.step_configurations import StepConfiguration
|
33
34
|
from zenml.config.step_run_info import StepRunInfo
|
34
35
|
from zenml.constants import (
|
@@ -36,10 +37,12 @@ from zenml.constants import (
|
|
36
37
|
ENV_ZENML_IGNORE_FAILURE_HOOK,
|
37
38
|
handle_bool_env_var,
|
38
39
|
)
|
40
|
+
from zenml.enums import ArtifactSaveType
|
39
41
|
from zenml.exceptions import StepInterfaceError
|
40
42
|
from zenml.logger import get_logger
|
41
43
|
from zenml.logging.step_logging import StepLogsStorageContext, redirected
|
42
44
|
from zenml.materializers.base_materializer import BaseMaterializer
|
45
|
+
from zenml.models.v2.core.step_run import StepRunInputResponse
|
43
46
|
from zenml.orchestrators.publish_utils import (
|
44
47
|
publish_step_run_metadata,
|
45
48
|
publish_successful_step_run,
|
@@ -98,7 +101,7 @@ class StepRunner:
|
|
98
101
|
self,
|
99
102
|
pipeline_run: "PipelineRunResponse",
|
100
103
|
step_run: "StepRunResponse",
|
101
|
-
input_artifacts: Dict[str,
|
104
|
+
input_artifacts: Dict[str, StepRunInputResponse],
|
102
105
|
output_artifact_uris: Dict[str, str],
|
103
106
|
step_run_info: StepRunInfo,
|
104
107
|
) -> None:
|
@@ -153,7 +156,7 @@ class StepRunner:
|
|
153
156
|
|
154
157
|
# Initialize the step context singleton
|
155
158
|
StepContext._clear()
|
156
|
-
StepContext(
|
159
|
+
step_context = StepContext(
|
157
160
|
pipeline_run=pipeline_run,
|
158
161
|
step_run=step_run,
|
159
162
|
output_materializers=output_materializers,
|
@@ -241,11 +244,16 @@ class StepRunner:
|
|
241
244
|
from zenml.orchestrators import step_run_utils
|
242
245
|
|
243
246
|
step_run_utils.link_output_artifacts_to_model_version(
|
244
|
-
artifacts=
|
247
|
+
artifacts={
|
248
|
+
k: [v] for k, v in output_artifacts.items()
|
249
|
+
},
|
245
250
|
output_configurations=step_run.config.outputs,
|
246
251
|
model_version=model_version,
|
247
252
|
)
|
248
253
|
finally:
|
254
|
+
step_context._cleanup_registry.execute_callbacks(
|
255
|
+
raise_on_exception=False
|
256
|
+
)
|
249
257
|
StepContext._clear() # Remove the step context singleton
|
250
258
|
|
251
259
|
# Update the status and output artifacts of the step run.
|
@@ -299,7 +307,7 @@ class StepRunner:
|
|
299
307
|
self,
|
300
308
|
args: List[str],
|
301
309
|
annotations: Dict[str, Any],
|
302
|
-
input_artifacts: Dict[str,
|
310
|
+
input_artifacts: Dict[str, StepRunInputResponse],
|
303
311
|
) -> Dict[str, Any]:
|
304
312
|
"""Parses the inputs for a step entrypoint function.
|
305
313
|
|
@@ -531,7 +539,7 @@ class StepRunner:
|
|
531
539
|
The IDs of the published output artifacts.
|
532
540
|
"""
|
533
541
|
step_context = get_step_context()
|
534
|
-
|
542
|
+
artifact_requests = []
|
535
543
|
|
536
544
|
for output_name, return_value in output_data.items():
|
537
545
|
data_type = type(return_value)
|
@@ -592,22 +600,25 @@ class StepRunner:
|
|
592
600
|
# Get full set of tags
|
593
601
|
tags = step_context.get_output_tags(output_name)
|
594
602
|
|
595
|
-
|
603
|
+
artifact_request = _store_artifact_data_and_prepare_request(
|
596
604
|
name=artifact_name,
|
597
605
|
data=return_value,
|
598
|
-
|
606
|
+
materializer_class=materializer_class,
|
599
607
|
uri=uri,
|
600
|
-
|
601
|
-
|
608
|
+
store_metadata=artifact_metadata_enabled,
|
609
|
+
store_visualizations=artifact_visualization_enabled,
|
602
610
|
has_custom_name=has_custom_name,
|
603
611
|
version=version,
|
604
612
|
tags=tags,
|
605
|
-
|
606
|
-
|
613
|
+
save_type=ArtifactSaveType.STEP_OUTPUT,
|
614
|
+
metadata=user_metadata,
|
607
615
|
)
|
608
|
-
|
616
|
+
artifact_requests.append(artifact_request)
|
609
617
|
|
610
|
-
|
618
|
+
responses = Client().zen_store.batch_create_artifact_versions(
|
619
|
+
artifact_requests
|
620
|
+
)
|
621
|
+
return dict(zip(output_data.keys(), responses))
|
611
622
|
|
612
623
|
def load_and_run_hook(
|
613
624
|
self,
|
zenml/orchestrators/utils.py
CHANGED
@@ -28,16 +28,16 @@ from zenml.constants import (
|
|
28
28
|
ENV_ZENML_DISABLE_CREDENTIALS_DISK_CACHING,
|
29
29
|
ENV_ZENML_SERVER,
|
30
30
|
ENV_ZENML_STORE_PREFIX,
|
31
|
-
PIPELINE_API_TOKEN_EXPIRES_MINUTES,
|
32
31
|
)
|
33
32
|
from zenml.enums import AuthScheme, StackComponentType, StoreType
|
34
33
|
from zenml.logger import get_logger
|
35
34
|
from zenml.stack import StackComponent
|
36
35
|
from zenml.utils.string_utils import format_name_template
|
37
36
|
|
37
|
+
logger = get_logger(__name__)
|
38
|
+
|
38
39
|
if TYPE_CHECKING:
|
39
40
|
from zenml.artifact_stores.base_artifact_store import BaseArtifactStore
|
40
|
-
from zenml.models import PipelineDeploymentResponse
|
41
41
|
|
42
42
|
|
43
43
|
def get_orchestrator_run_name(pipeline_name: str) -> str:
|
@@ -80,16 +80,23 @@ def is_setting_enabled(
|
|
80
80
|
|
81
81
|
|
82
82
|
def get_config_environment_vars(
|
83
|
-
|
83
|
+
schedule_id: Optional[UUID] = None,
|
84
|
+
pipeline_run_id: Optional[UUID] = None,
|
85
|
+
step_run_id: Optional[UUID] = None,
|
84
86
|
) -> Dict[str, str]:
|
85
87
|
"""Gets environment variables to set for mirroring the active config.
|
86
88
|
|
87
|
-
If a pipeline
|
88
|
-
|
89
|
-
|
89
|
+
If a schedule ID, pipeline run ID or step run ID is given, and the current
|
90
|
+
client is not authenticated to a server with an API key, the environment
|
91
|
+
variables will be updated to include a newly generated workload API token
|
92
|
+
that will be valid for the duration of the schedule, pipeline run, or step
|
93
|
+
run instead of the current API token used to authenticate the client.
|
90
94
|
|
91
95
|
Args:
|
92
|
-
|
96
|
+
schedule_id: Optional schedule ID to use to generate a new API token.
|
97
|
+
pipeline_run_id: Optional pipeline run ID to use to generate a new API
|
98
|
+
token.
|
99
|
+
step_run_id: Optional step run ID to use to generate a new API token.
|
93
100
|
|
94
101
|
Returns:
|
95
102
|
Environment variable dict.
|
@@ -107,34 +114,46 @@ def get_config_environment_vars(
|
|
107
114
|
):
|
108
115
|
credentials_store = get_credentials_store()
|
109
116
|
url = global_config.store_configuration.url
|
110
|
-
api_key = credentials_store.get_api_key(url)
|
111
117
|
api_token = credentials_store.get_token(url, allow_expired=False)
|
112
|
-
if
|
113
|
-
|
114
|
-
|
115
|
-
#
|
116
|
-
#
|
117
|
-
# valid for the duration of the pipeline run.
|
118
|
+
if schedule_id or pipeline_run_id or step_run_id:
|
119
|
+
# When connected to an authenticated ZenML server, if a schedule ID,
|
120
|
+
# pipeline run ID or step run ID is supplied, we need to fetch a new
|
121
|
+
# workload API token scoped to the schedule, pipeline run or step
|
122
|
+
# run.
|
118
123
|
assert isinstance(global_config.zen_store, RestZenStore)
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
124
|
+
|
125
|
+
# If only a schedule is given, the pipeline run credentials will
|
126
|
+
# be valid for the entire duration of the schedule.
|
127
|
+
api_key = credentials_store.get_api_key(url)
|
128
|
+
if not api_key and not pipeline_run_id and not step_run_id:
|
129
|
+
logger.warning(
|
130
|
+
"An API token without an expiration time will be generated "
|
131
|
+
"and used to run this pipeline on a schedule. This is very "
|
132
|
+
"insecure because the API token will be valid for the "
|
133
|
+
"entire lifetime of the schedule and can be used to access "
|
134
|
+
"your user account if accidentally leaked. When deploying "
|
135
|
+
"a pipeline on a schedule, it is strongly advised to use a "
|
136
|
+
"service account API key to authenticate to the ZenML "
|
137
|
+
"server instead of your regular user account. For more "
|
138
|
+
"information, see "
|
139
|
+
"https://docs.zenml.io/how-to/connecting-to-zenml/connect-with-a-service-account"
|
140
|
+
)
|
141
|
+
|
142
|
+
# The schedule, pipeline run or step run credentials are scoped to
|
143
|
+
# the schedule, pipeline run or step run and will only be valid for
|
144
|
+
# the duration of the schedule/pipeline run/step run.
|
129
145
|
new_api_token = global_config.zen_store.get_api_token(
|
130
|
-
pipeline_id=pipeline_id,
|
131
146
|
schedule_id=schedule_id,
|
132
|
-
|
147
|
+
pipeline_run_id=pipeline_run_id,
|
148
|
+
step_run_id=step_run_id,
|
133
149
|
)
|
150
|
+
|
134
151
|
environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
|
135
152
|
new_api_token
|
136
153
|
)
|
137
154
|
elif api_token:
|
155
|
+
# For all other cases, the pipeline run environment is configured
|
156
|
+
# with the current access token.
|
138
157
|
environment_vars[ENV_ZENML_STORE_PREFIX + "API_TOKEN"] = (
|
139
158
|
api_token.access_token
|
140
159
|
)
|
zenml/stack/flavor.py
CHANGED
@@ -16,6 +16,7 @@
|
|
16
16
|
from abc import abstractmethod
|
17
17
|
from typing import Any, Dict, Optional, Type, cast
|
18
18
|
|
19
|
+
from zenml.client import Client
|
19
20
|
from zenml.enums import StackComponentType
|
20
21
|
from zenml.models import (
|
21
22
|
FlavorRequest,
|
@@ -146,9 +147,6 @@ class Flavor:
|
|
146
147
|
Returns:
|
147
148
|
The model.
|
148
149
|
"""
|
149
|
-
from zenml.client import Client
|
150
|
-
|
151
|
-
client = Client()
|
152
150
|
connector_requirements = self.service_connector_requirements
|
153
151
|
connector_type = (
|
154
152
|
connector_requirements.connector_type
|
@@ -165,10 +163,16 @@ class Flavor:
|
|
165
163
|
if connector_requirements
|
166
164
|
else None
|
167
165
|
)
|
166
|
+
user = None
|
167
|
+
workspace = None
|
168
|
+
if is_custom:
|
169
|
+
user = Client().active_user.id
|
170
|
+
workspace = Client().active_workspace.id
|
171
|
+
|
168
172
|
model_class = FlavorRequest if is_custom else InternalFlavorRequest
|
169
173
|
model = model_class(
|
170
|
-
user=
|
171
|
-
workspace=
|
174
|
+
user=user,
|
175
|
+
workspace=workspace,
|
172
176
|
name=self.name,
|
173
177
|
type=self.type,
|
174
178
|
source=source_utils.resolve(self.__class__).import_path,
|
@@ -71,7 +71,8 @@ of any potential costs:
|
|
71
71
|
|
72
72
|
- An S3 bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/s3).
|
73
73
|
- An ECR repository registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/aws).
|
74
|
-
- Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
|
74
|
+
- Sagemaker registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/sagemaker)
|
75
|
+
as well as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/sagemaker).
|
75
76
|
- An IAM user and IAM role with the minimum necessary permissions to access the
|
76
77
|
above resources.
|
77
78
|
- An AWS access key used to give access to ZenML to connect to the above
|
@@ -257,13 +258,29 @@ console.
|
|
257
258
|
|
258
259
|
config: Optional[str] = None
|
259
260
|
if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
|
260
|
-
config = f"""
|
261
|
-
|
261
|
+
config = f"""terraform {{
|
262
|
+
required_providers {{
|
263
|
+
aws = {{
|
264
|
+
source = "hashicorp/aws"
|
265
|
+
}}
|
266
|
+
zenml = {{
|
267
|
+
source = "zenml-io/zenml"
|
268
|
+
}}
|
269
|
+
}}
|
270
|
+
}}
|
262
271
|
|
272
|
+
provider "aws" {{
|
263
273
|
region = "{self.location or "eu-central-1"}"
|
264
|
-
|
265
|
-
|
266
|
-
|
274
|
+
}}
|
275
|
+
|
276
|
+
provider "zenml" {{
|
277
|
+
server_url = "{self.zenml_server_url}"
|
278
|
+
api_token = "{self.zenml_server_api_token}"
|
279
|
+
}}
|
280
|
+
|
281
|
+
module "zenml_stack" {{
|
282
|
+
source = "zenml-io/zenml-stack/aws"
|
283
|
+
|
267
284
|
zenml_stack_name = "{self.stack_name}"
|
268
285
|
zenml_stack_deployment = "{self.deployment_type}"
|
269
286
|
}}
|
@@ -259,14 +259,37 @@ ZenML's access to your Azure subscription.
|
|
259
259
|
The configuration or script to deploy the ZenML stack to the
|
260
260
|
specified cloud provider.
|
261
261
|
"""
|
262
|
-
config = f"""
|
262
|
+
config = f"""terraform {{
|
263
|
+
required_providers {{
|
264
|
+
azurerm = {{
|
265
|
+
source = "hashicorp/azurerm"
|
266
|
+
}}
|
267
|
+
azuread = {{
|
268
|
+
source = "hashicorp/azuread"
|
269
|
+
}}
|
270
|
+
zenml = {{
|
271
|
+
source = "zenml-io/zenml"
|
272
|
+
}}
|
273
|
+
}}
|
274
|
+
}}
|
275
|
+
|
276
|
+
provider "azurerm" {{
|
277
|
+
features {{
|
278
|
+
resource_group {{
|
279
|
+
prevent_deletion_if_contains_resources = false
|
280
|
+
}}
|
281
|
+
}}
|
282
|
+
}}
|
283
|
+
|
284
|
+
provider "zenml" {{
|
285
|
+
server_url = "{self.zenml_server_url}"
|
286
|
+
api_token = "{self.zenml_server_api_token}"
|
287
|
+
}}
|
288
|
+
|
289
|
+
module "zenml_stack" {{
|
263
290
|
source = "zenml-io/zenml-stack/azure"
|
264
291
|
|
265
292
|
location = "{self.location or "eastus"}"
|
266
|
-
orchestrator = "azureml"
|
267
|
-
zenml_server_url = "{self.zenml_server_url}"
|
268
|
-
zenml_api_key = ""
|
269
|
-
zenml_api_token = "{self.zenml_server_api_token}"
|
270
293
|
zenml_stack_name = "{self.stack_name}"
|
271
294
|
zenml_stack_deployment = "{self.deployment_type}"
|
272
295
|
}}
|
@@ -70,11 +70,12 @@ and are aware of any potential costs:
|
|
70
70
|
|
71
71
|
- A GCS bucket registered as a [ZenML artifact store](https://docs.zenml.io/stack-components/artifact-stores/gcp).
|
72
72
|
- A Google Artifact Registry registered as a [ZenML container registry](https://docs.zenml.io/stack-components/container-registries/gcp).
|
73
|
-
- Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
|
73
|
+
- Vertex AI registered as a [ZenML orchestrator](https://docs.zenml.io/stack-components/orchestrators/vertex)
|
74
|
+
and as a [ZenML step operator](https://docs.zenml.io/stack-components/step-operators/vertex).
|
74
75
|
- GCP Cloud Build registered as a [ZenML image builder](https://docs.zenml.io/stack-components/image-builders/gcp).
|
75
76
|
- A GCP Service Account with the minimum necessary permissions to access the
|
76
77
|
above resources.
|
77
|
-
-
|
78
|
+
- A GCP Service Account access key used to give access to ZenML to connect to
|
78
79
|
the above resources through a [ZenML service connector](https://docs.zenml.io/how-to/auth-management/gcp-service-connector).
|
79
80
|
|
80
81
|
The Deployment Manager deployment will automatically create a GCP Service
|
@@ -259,14 +260,30 @@ GCP project and to clean up the resources created by the stack by using
|
|
259
260
|
)
|
260
261
|
|
261
262
|
if self.deployment_type == STACK_DEPLOYMENT_TERRAFORM:
|
262
|
-
config = f"""
|
263
|
+
config = f"""terraform {{
|
264
|
+
required_providers {{
|
265
|
+
google = {{
|
266
|
+
source = "hashicorp/google"
|
267
|
+
}}
|
268
|
+
zenml = {{
|
269
|
+
source = "zenml-io/zenml"
|
270
|
+
}}
|
271
|
+
}}
|
272
|
+
}}
|
273
|
+
|
274
|
+
provider "google" {{
|
275
|
+
region = "{self.location or "europe-west3"}"
|
276
|
+
project = your GCP project name
|
277
|
+
}}
|
278
|
+
|
279
|
+
provider "zenml" {{
|
280
|
+
server_url = "{self.zenml_server_url}"
|
281
|
+
api_token = "{self.zenml_server_api_token}"
|
282
|
+
}}
|
283
|
+
|
284
|
+
module "zenml_stack" {{
|
263
285
|
source = "zenml-io/zenml-stack/gcp"
|
264
286
|
|
265
|
-
project_id = "my-gcp-project"
|
266
|
-
region = "{self.location or "europe-west3"}"
|
267
|
-
zenml_server_url = "{self.zenml_server_url}"
|
268
|
-
zenml_api_key = ""
|
269
|
-
zenml_api_token = "{self.zenml_server_api_token}"
|
270
287
|
zenml_stack_name = "{self.stack_name}"
|
271
288
|
zenml_stack_deployment = "{self.deployment_type}"
|
272
289
|
}}
|
@@ -219,16 +219,14 @@ class ZenMLCloudStackDeployment(BaseModel):
|
|
219
219
|
if stack.labels.get("zenml:deployment") != self.deployment_type:
|
220
220
|
continue
|
221
221
|
|
222
|
-
|
223
|
-
StackComponentType.ARTIFACT_STORE
|
224
|
-
][0]
|
222
|
+
orchestrator = stack.components[StackComponentType.ORCHESTRATOR][0]
|
225
223
|
|
226
|
-
if not
|
224
|
+
if not orchestrator.connector:
|
227
225
|
continue
|
228
226
|
|
229
227
|
return DeployedStack(
|
230
228
|
stack=stack,
|
231
|
-
service_connector=
|
229
|
+
service_connector=orchestrator.connector,
|
232
230
|
)
|
233
231
|
|
234
232
|
return None
|