zenml-nightly 0.68.1.dev20241103__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +77 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/artifacts/external_artifact.py +2 -1
- zenml/artifacts/{load_directory_materializer.py → preexisting_data_materializer.py} +8 -9
- zenml/artifacts/utils.py +139 -80
- zenml/cli/base.py +4 -4
- zenml/cli/model.py +1 -6
- zenml/cli/stack.py +1 -0
- zenml/client.py +29 -74
- zenml/config/server_config.py +17 -1
- zenml/constants.py +2 -7
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/enums.py +20 -4
- zenml/exceptions.py +4 -0
- zenml/integrations/__init__.py +3 -1
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
- zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
- zenml/integrations/constants.py +1 -0
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
- zenml/integrations/evidently/__init__.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +8 -12
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +2 -5
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -3
- zenml/integrations/langchain/__init__.py +2 -1
- zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
- zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
- zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
- zenml/integrations/openai/__init__.py +1 -1
- zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
- zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
- zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
- zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
- zenml/integrations/tensorflow/materializers/keras_materializer.py +11 -22
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +8 -15
- zenml/integrations/vllm/__init__.py +50 -0
- zenml/integrations/vllm/flavors/__init__.py +21 -0
- zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
- zenml/integrations/vllm/model_deployers/__init__.py +19 -0
- zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
- zenml/integrations/vllm/services/__init__.py +19 -0
- zenml/integrations/vllm/services/vllm_deployment.py +206 -0
- zenml/integrations/whylogs/materializers/whylogs_materializer.py +11 -18
- zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
- zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
- zenml/materializers/base_materializer.py +68 -1
- zenml/metadata/lazy_load.py +20 -7
- zenml/model/model.py +17 -64
- zenml/model/utils.py +5 -0
- zenml/models/__init__.py +0 -12
- zenml/models/v2/base/filter.py +121 -8
- zenml/models/v2/core/artifact_version.py +42 -7
- zenml/models/v2/core/model_version.py +26 -5
- zenml/models/v2/core/pipeline_run.py +25 -6
- zenml/models/v2/core/run_metadata.py +2 -217
- zenml/models/v2/core/step_run.py +62 -24
- zenml/orchestrators/base_orchestrator.py +12 -1
- zenml/orchestrators/input_utils.py +44 -19
- zenml/orchestrators/step_launcher.py +4 -3
- zenml/orchestrators/step_run_utils.py +19 -15
- zenml/orchestrators/step_runner.py +25 -14
- zenml/orchestrators/utils.py +45 -26
- zenml/stack/flavor.py +9 -5
- zenml/stack_deployments/aws_stack_deployment.py +23 -6
- zenml/stack_deployments/azure_stack_deployment.py +28 -5
- zenml/stack_deployments/gcp_stack_deployment.py +25 -8
- zenml/stack_deployments/stack_deployment.py +3 -5
- zenml/steps/base_step.py +1 -1
- zenml/steps/entrypoint_function_utils.py +3 -5
- zenml/steps/step_context.py +5 -2
- zenml/steps/utils.py +13 -2
- zenml/utils/callback_registry.py +71 -0
- zenml/utils/metadata_utils.py +335 -0
- zenml/zen_server/auth.py +221 -3
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
- zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
- zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
- zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
- zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
- zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
- zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
- zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
- zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
- zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
- zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
- zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
- zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
- zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
- zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
- zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
- zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
- zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
- zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
- zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
- zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
- zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
- zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
- zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +2 -0
- zenml/zen_server/jwt.py +30 -13
- zenml/zen_server/rbac/endpoint_utils.py +43 -1
- zenml/zen_server/rbac/utils.py +0 -2
- zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
- zenml/zen_server/routers/auth_endpoints.py +134 -102
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/workspaces_endpoints.py +3 -4
- zenml/zen_server/template_execution/utils.py +14 -16
- zenml/zen_server/utils.py +27 -0
- zenml/zen_server/zen_server_api.py +6 -3
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/rest_zen_store.py +68 -64
- zenml/zen_stores/schemas/artifact_schemas.py +8 -1
- zenml/zen_stores/schemas/model_schemas.py +27 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
- zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
- zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
- zenml/zen_stores/schemas/step_run_schemas.py +18 -10
- zenml/zen_stores/sql_zen_store.py +283 -219
- zenml/zen_stores/zen_store_interface.py +15 -42
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +226 -203
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
- zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
- zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
- zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
- zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
- zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
- zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
- zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
- zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
- zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
zenml/client.py
CHANGED
@@ -136,9 +136,7 @@ from zenml.models import (
|
|
136
136
|
PipelineResponse,
|
137
137
|
PipelineRunFilter,
|
138
138
|
PipelineRunResponse,
|
139
|
-
RunMetadataFilter,
|
140
139
|
RunMetadataRequest,
|
141
|
-
RunMetadataResponse,
|
142
140
|
RunTemplateFilter,
|
143
141
|
RunTemplateRequest,
|
144
142
|
RunTemplateResponse,
|
@@ -190,6 +188,7 @@ from zenml.models import (
|
|
190
188
|
WorkspaceResponse,
|
191
189
|
WorkspaceUpdate,
|
192
190
|
)
|
191
|
+
from zenml.models.v2.core.step_run import StepRunUpdate
|
193
192
|
from zenml.services.service import ServiceConfig
|
194
193
|
from zenml.services.service_status import ServiceState
|
195
194
|
from zenml.services.service_type import ServiceType
|
@@ -3427,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
|
|
3427
3426
|
Args:
|
3428
3427
|
id_or_prefix: The id or id prefix of the deployment.
|
3429
3428
|
"""
|
3430
|
-
deployment = self.get_deployment(
|
3429
|
+
deployment = self.get_deployment(
|
3430
|
+
id_or_prefix=id_or_prefix, hydrate=False
|
3431
|
+
)
|
3431
3432
|
self.zen_store.delete_deployment(deployment_id=deployment.id)
|
3432
3433
|
|
3433
3434
|
# ------------------------------ Run templates -----------------------------
|
@@ -3795,6 +3796,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3795
3796
|
templatable: Optional[bool] = None,
|
3796
3797
|
tag: Optional[str] = None,
|
3797
3798
|
user: Optional[Union[UUID, str]] = None,
|
3799
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
3798
3800
|
pipeline: Optional[Union[UUID, str]] = None,
|
3799
3801
|
code_repository: Optional[Union[UUID, str]] = None,
|
3800
3802
|
model: Optional[Union[UUID, str]] = None,
|
@@ -3834,6 +3836,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3834
3836
|
templatable: If the runs should be templatable or not.
|
3835
3837
|
tag: Tag to filter by.
|
3836
3838
|
user: The name/ID of the user to filter by.
|
3839
|
+
run_metadata: The run_metadata of the run to filter by.
|
3837
3840
|
pipeline: The name/ID of the pipeline to filter by.
|
3838
3841
|
code_repository: Filter by code repository name/ID.
|
3839
3842
|
model: Filter by model name/ID.
|
@@ -3873,6 +3876,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3873
3876
|
tag=tag,
|
3874
3877
|
unlisted=unlisted,
|
3875
3878
|
user=user,
|
3879
|
+
run_metadata=run_metadata,
|
3876
3880
|
pipeline=pipeline,
|
3877
3881
|
code_repository=code_repository,
|
3878
3882
|
stack=stack,
|
@@ -4166,6 +4170,8 @@ class Client(metaclass=ClientMetaClass):
|
|
4166
4170
|
Returns:
|
4167
4171
|
The artifact version.
|
4168
4172
|
"""
|
4173
|
+
from zenml import get_step_context
|
4174
|
+
|
4169
4175
|
if cll := client_lazy_loader(
|
4170
4176
|
method_name="get_artifact_version",
|
4171
4177
|
name_id_or_prefix=name_id_or_prefix,
|
@@ -4173,13 +4179,26 @@ class Client(metaclass=ClientMetaClass):
|
|
4173
4179
|
hydrate=hydrate,
|
4174
4180
|
):
|
4175
4181
|
return cll # type: ignore[return-value]
|
4176
|
-
|
4182
|
+
|
4183
|
+
artifact = self._get_entity_version_by_id_or_name_or_prefix(
|
4177
4184
|
get_method=self.zen_store.get_artifact_version,
|
4178
4185
|
list_method=self.list_artifact_versions,
|
4179
4186
|
name_id_or_prefix=name_id_or_prefix,
|
4180
4187
|
version=version,
|
4181
4188
|
hydrate=hydrate,
|
4182
4189
|
)
|
4190
|
+
try:
|
4191
|
+
step_run = get_step_context().step_run
|
4192
|
+
client = Client()
|
4193
|
+
client.zen_store.update_run_step(
|
4194
|
+
step_run_id=step_run.id,
|
4195
|
+
step_run_update=StepRunUpdate(
|
4196
|
+
loaded_artifact_versions={artifact.name: artifact.id}
|
4197
|
+
),
|
4198
|
+
)
|
4199
|
+
except RuntimeError:
|
4200
|
+
pass # Cannot link to step run if called outside a step
|
4201
|
+
return artifact
|
4183
4202
|
|
4184
4203
|
def list_artifact_versions(
|
4185
4204
|
self,
|
@@ -4206,6 +4225,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4206
4225
|
user: Optional[Union[UUID, str]] = None,
|
4207
4226
|
model: Optional[Union[UUID, str]] = None,
|
4208
4227
|
pipeline_run: Optional[Union[UUID, str]] = None,
|
4228
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
4209
4229
|
tag: Optional[str] = None,
|
4210
4230
|
hydrate: bool = False,
|
4211
4231
|
) -> Page[ArtifactVersionResponse]:
|
@@ -4237,6 +4257,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4237
4257
|
user: Filter by user name or ID.
|
4238
4258
|
model: Filter by model name or ID.
|
4239
4259
|
pipeline_run: Filter by pipeline run name or ID.
|
4260
|
+
run_metadata: Filter by run metadata.
|
4240
4261
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
4241
4262
|
by including metadata fields in the response.
|
4242
4263
|
|
@@ -4417,7 +4438,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4417
4438
|
resource_id: UUID,
|
4418
4439
|
resource_type: MetadataResourceTypes,
|
4419
4440
|
stack_component_id: Optional[UUID] = None,
|
4420
|
-
) ->
|
4441
|
+
) -> None:
|
4421
4442
|
"""Create run metadata.
|
4422
4443
|
|
4423
4444
|
Args:
|
@@ -4430,7 +4451,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4430
4451
|
the metadata.
|
4431
4452
|
|
4432
4453
|
Returns:
|
4433
|
-
|
4454
|
+
None
|
4434
4455
|
"""
|
4435
4456
|
from zenml.metadata.metadata_types import get_metadata_type
|
4436
4457
|
|
@@ -4465,74 +4486,8 @@ class Client(metaclass=ClientMetaClass):
|
|
4465
4486
|
values=values,
|
4466
4487
|
types=types,
|
4467
4488
|
)
|
4468
|
-
|
4469
|
-
|
4470
|
-
def list_run_metadata(
|
4471
|
-
self,
|
4472
|
-
sort_by: str = "created",
|
4473
|
-
page: int = PAGINATION_STARTING_PAGE,
|
4474
|
-
size: int = PAGE_SIZE_DEFAULT,
|
4475
|
-
logical_operator: LogicalOperators = LogicalOperators.AND,
|
4476
|
-
id: Optional[Union[UUID, str]] = None,
|
4477
|
-
created: Optional[Union[datetime, str]] = None,
|
4478
|
-
updated: Optional[Union[datetime, str]] = None,
|
4479
|
-
workspace_id: Optional[UUID] = None,
|
4480
|
-
user_id: Optional[UUID] = None,
|
4481
|
-
resource_id: Optional[UUID] = None,
|
4482
|
-
resource_type: Optional[MetadataResourceTypes] = None,
|
4483
|
-
stack_component_id: Optional[UUID] = None,
|
4484
|
-
key: Optional[str] = None,
|
4485
|
-
value: Optional["MetadataType"] = None,
|
4486
|
-
type: Optional[str] = None,
|
4487
|
-
hydrate: bool = False,
|
4488
|
-
) -> Page[RunMetadataResponse]:
|
4489
|
-
"""List run metadata.
|
4490
|
-
|
4491
|
-
Args:
|
4492
|
-
sort_by: The field to sort the results by.
|
4493
|
-
page: The page number to return.
|
4494
|
-
size: The number of results to return per page.
|
4495
|
-
logical_operator: The logical operator to use for filtering.
|
4496
|
-
id: The ID of the metadata.
|
4497
|
-
created: The creation time of the metadata.
|
4498
|
-
updated: The last update time of the metadata.
|
4499
|
-
workspace_id: The ID of the workspace the metadata belongs to.
|
4500
|
-
user_id: The ID of the user that created the metadata.
|
4501
|
-
resource_id: The ID of the resource the metadata belongs to.
|
4502
|
-
resource_type: The type of the resource the metadata belongs to.
|
4503
|
-
stack_component_id: The ID of the stack component that produced
|
4504
|
-
the metadata.
|
4505
|
-
key: The key of the metadata.
|
4506
|
-
value: The value of the metadata.
|
4507
|
-
type: The type of the metadata.
|
4508
|
-
hydrate: Flag deciding whether to hydrate the output model(s)
|
4509
|
-
by including metadata fields in the response.
|
4510
|
-
|
4511
|
-
Returns:
|
4512
|
-
The run metadata.
|
4513
|
-
"""
|
4514
|
-
metadata_filter_model = RunMetadataFilter(
|
4515
|
-
sort_by=sort_by,
|
4516
|
-
page=page,
|
4517
|
-
size=size,
|
4518
|
-
logical_operator=logical_operator,
|
4519
|
-
id=id,
|
4520
|
-
created=created,
|
4521
|
-
updated=updated,
|
4522
|
-
workspace_id=workspace_id,
|
4523
|
-
user_id=user_id,
|
4524
|
-
resource_id=resource_id,
|
4525
|
-
resource_type=resource_type,
|
4526
|
-
stack_component_id=stack_component_id,
|
4527
|
-
key=key,
|
4528
|
-
value=value,
|
4529
|
-
type=type,
|
4530
|
-
)
|
4531
|
-
metadata_filter_model.set_scope_workspace(self.active_workspace.id)
|
4532
|
-
return self.zen_store.list_run_metadata(
|
4533
|
-
metadata_filter_model,
|
4534
|
-
hydrate=hydrate,
|
4535
|
-
)
|
4489
|
+
self.zen_store.create_run_metadata(run_metadata)
|
4490
|
+
return None
|
4536
4491
|
|
4537
4492
|
# -------------------------------- Secrets ---------------------------------
|
4538
4493
|
|
zenml/config/server_config.py
CHANGED
@@ -19,13 +19,14 @@ from secrets import token_hex
|
|
19
19
|
from typing import Any, Dict, List, Optional, Union
|
20
20
|
from uuid import UUID
|
21
21
|
|
22
|
-
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
22
|
+
from pydantic import BaseModel, ConfigDict, Field, PositiveInt, model_validator
|
23
23
|
|
24
24
|
from zenml.constants import (
|
25
25
|
DEFAULT_ZENML_JWT_TOKEN_ALGORITHM,
|
26
26
|
DEFAULT_ZENML_JWT_TOKEN_LEEWAY,
|
27
27
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING,
|
28
28
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT,
|
29
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME,
|
29
30
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY,
|
30
31
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE,
|
31
32
|
DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS,
|
@@ -119,6 +120,8 @@ class ServerConfiguration(BaseModel):
|
|
119
120
|
time of the JWT tokens issued to clients after they have
|
120
121
|
authenticated with the ZenML server using an OAuth 2.0 device
|
121
122
|
that has been marked as trusted.
|
123
|
+
generic_api_token_lifetime: The lifetime in seconds that generic
|
124
|
+
short-lived API tokens issued for automation purposes are valid.
|
122
125
|
external_login_url: The login URL of an external authenticator service
|
123
126
|
to use with the `EXTERNAL` authentication scheme.
|
124
127
|
external_user_info_url: The user info URL of an external authenticator
|
@@ -230,6 +233,12 @@ class ServerConfiguration(BaseModel):
|
|
230
233
|
deployment.
|
231
234
|
max_request_body_size_in_bytes: The maximum size of the request body in
|
232
235
|
bytes. If not specified, the default value of 256 Kb will be used.
|
236
|
+
memcache_max_capacity: The maximum number of entries that the memory
|
237
|
+
cache can hold. If not specified, the default value of 1000 will be
|
238
|
+
used.
|
239
|
+
memcache_default_expiry: The default expiry time in seconds for cache
|
240
|
+
entries. If not specified, the default value of 30 seconds will be
|
241
|
+
used.
|
233
242
|
"""
|
234
243
|
|
235
244
|
deployment_type: ServerDeploymentType = ServerDeploymentType.OTHER
|
@@ -257,6 +266,10 @@ class ServerConfiguration(BaseModel):
|
|
257
266
|
device_expiration_minutes: Optional[int] = None
|
258
267
|
trusted_device_expiration_minutes: Optional[int] = None
|
259
268
|
|
269
|
+
generic_api_token_lifetime: PositiveInt = (
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME
|
271
|
+
)
|
272
|
+
|
260
273
|
external_login_url: Optional[str] = None
|
261
274
|
external_user_info_url: Optional[str] = None
|
262
275
|
external_cookie_name: Optional[str] = None
|
@@ -321,6 +334,9 @@ class ServerConfiguration(BaseModel):
|
|
321
334
|
DEFAULT_ZENML_SERVER_MAX_REQUEST_BODY_SIZE_IN_BYTES
|
322
335
|
)
|
323
336
|
|
337
|
+
memcache_max_capacity: int = 1000
|
338
|
+
memcache_default_expiry: int = 30
|
339
|
+
|
324
340
|
_deployment_id: Optional[UUID] = None
|
325
341
|
|
326
342
|
@model_validator(mode="before")
|
zenml/constants.py
CHANGED
@@ -167,9 +167,6 @@ ENV_ZENML_SERVER = "ZENML_SERVER"
|
|
167
167
|
ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
|
168
168
|
ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS"
|
169
169
|
ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
|
170
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
|
171
|
-
"ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES"
|
172
|
-
)
|
173
170
|
ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
|
174
171
|
ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
|
175
172
|
ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
|
@@ -270,6 +267,7 @@ ZENML_API_KEY_PREFIX = "ZENKEY_"
|
|
270
267
|
DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48 # 48 hours
|
271
268
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
|
272
269
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY = 1000
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME = 60 * 60 # 1 hour
|
273
271
|
|
274
272
|
DEFAULT_ZENML_SERVER_SECURE_HEADERS_HSTS = (
|
275
273
|
"max-age=63072000; includeSubdomains"
|
@@ -338,6 +336,7 @@ ARTIFACTS = "/artifacts"
|
|
338
336
|
ARTIFACT_VERSIONS = "/artifact_versions"
|
339
337
|
ARTIFACT_VISUALIZATIONS = "/artifact_visualizations"
|
340
338
|
AUTH = "/auth"
|
339
|
+
BATCH = "/batch"
|
341
340
|
CODE_REFERENCES = "/code_references"
|
342
341
|
CODE_REPOSITORIES = "/code_repositories"
|
343
342
|
COMPONENT_TYPES = "/component-types"
|
@@ -409,10 +408,6 @@ MODEL_METADATA_YAML_FILE_NAME = "model_metadata.yaml"
|
|
409
408
|
|
410
409
|
# orchestrator constants
|
411
410
|
ORCHESTRATOR_DOCKER_IMAGE_KEY = "orchestrator"
|
412
|
-
PIPELINE_API_TOKEN_EXPIRES_MINUTES = handle_int_env_var(
|
413
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES,
|
414
|
-
default=60 * 24, # 24 hours
|
415
|
-
)
|
416
411
|
|
417
412
|
# Secret constants
|
418
413
|
SECRET_VALUES = "values"
|
@@ -175,7 +175,7 @@ class BaseDataValidator(StackComponent):
|
|
175
175
|
|
176
176
|
This method should be implemented by data validators that support
|
177
177
|
running model validation checks (e.g. confusion matrix validation,
|
178
|
-
performance reports, model error
|
178
|
+
performance reports, model error analyzes, etc).
|
179
179
|
|
180
180
|
Unlike `data_validation`, model validation checks require that a model
|
181
181
|
be present as an active component during the validation process.
|
@@ -184,7 +184,7 @@ class BaseDataValidator(StackComponent):
|
|
184
184
|
accommodate different categories of data validation tests, e.g.:
|
185
185
|
|
186
186
|
* single dataset tests: confusion matrix validation,
|
187
|
-
performance reports, model error
|
187
|
+
performance reports, model error analyzes, etc
|
188
188
|
* model comparison tests: tests that identify changes in a model
|
189
189
|
behavior by comparing how it performs on two different datasets.
|
190
190
|
|
zenml/enums.py
CHANGED
@@ -34,15 +34,23 @@ class ArtifactType(StrEnum):
|
|
34
34
|
class StepRunInputArtifactType(StrEnum):
|
35
35
|
"""All possible types of a step run input artifact."""
|
36
36
|
|
37
|
-
|
37
|
+
STEP_OUTPUT = (
|
38
|
+
"step_output" # input argument that is the output of a previous step
|
39
|
+
)
|
38
40
|
MANUAL = "manual" # manually loaded via `zenml.load_artifact()`
|
41
|
+
EXTERNAL = "external" # loaded via `ExternalArtifact(value=...)`
|
42
|
+
LAZY_LOADED = "lazy" # loaded via various lazy methods
|
39
43
|
|
40
44
|
|
41
|
-
class
|
42
|
-
"""All possible types of
|
45
|
+
class ArtifactSaveType(StrEnum):
|
46
|
+
"""All possible method types of how artifact versions can be saved."""
|
43
47
|
|
44
|
-
|
48
|
+
STEP_OUTPUT = "step_output" # output of the current step
|
45
49
|
MANUAL = "manual" # manually saved via `zenml.save_artifact()`
|
50
|
+
PREEXISTING = "preexisting" # register via `zenml.register_artifact()`
|
51
|
+
EXTERNAL = (
|
52
|
+
"external" # saved via `zenml.ExternalArtifact.upload_by_value()`
|
53
|
+
)
|
46
54
|
|
47
55
|
|
48
56
|
class VisualizationType(StrEnum):
|
@@ -237,6 +245,13 @@ class OAuthDeviceStatus(StrEnum):
|
|
237
245
|
LOCKED = "locked"
|
238
246
|
|
239
247
|
|
248
|
+
class APITokenType(StrEnum):
|
249
|
+
"""The API token type."""
|
250
|
+
|
251
|
+
GENERIC = "generic"
|
252
|
+
WORKLOAD = "workload"
|
253
|
+
|
254
|
+
|
240
255
|
class GenericFilterOps(StrEnum):
|
241
256
|
"""Ops for all filters for string values on list methods."""
|
242
257
|
|
@@ -245,6 +260,7 @@ class GenericFilterOps(StrEnum):
|
|
245
260
|
CONTAINS = "contains"
|
246
261
|
STARTSWITH = "startswith"
|
247
262
|
ENDSWITH = "endswith"
|
263
|
+
ONEOF = "oneof"
|
248
264
|
GTE = "gte"
|
249
265
|
GT = "gt"
|
250
266
|
LTE = "lte"
|
zenml/exceptions.py
CHANGED
@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
|
|
168
168
|
"""Raised when trying to register an entity that already exists."""
|
169
169
|
|
170
170
|
|
171
|
+
class EntityCreationError(ZenMLBaseException, RuntimeError):
|
172
|
+
"""Raised when failing to create an entity."""
|
173
|
+
|
174
|
+
|
171
175
|
class ActionExistsError(EntityExistsError):
|
172
176
|
"""Raised when registering an action with a name that already exists."""
|
173
177
|
|
zenml/integrations/__init__.py
CHANGED
@@ -45,6 +45,7 @@ from zenml.integrations.kubernetes import KubernetesIntegration # noqa
|
|
45
45
|
from zenml.integrations.label_studio import LabelStudioIntegration # noqa
|
46
46
|
from zenml.integrations.langchain import LangchainIntegration # noqa
|
47
47
|
from zenml.integrations.lightgbm import LightGBMIntegration # noqa
|
48
|
+
|
48
49
|
# from zenml.integrations.llama_index import LlamaIndexIntegration # noqa
|
49
50
|
from zenml.integrations.mlflow import MlflowIntegration # noqa
|
50
51
|
from zenml.integrations.neptune import NeptuneIntegration # noqa
|
@@ -52,7 +53,7 @@ from zenml.integrations.neural_prophet import NeuralProphetIntegration # noqa
|
|
52
53
|
from zenml.integrations.numpy import NumpyIntegration # noqa
|
53
54
|
from zenml.integrations.openai import OpenAIIntegration # noqa
|
54
55
|
from zenml.integrations.pandas import PandasIntegration # noqa
|
55
|
-
from zenml.integrations.pigeon import PigeonIntegration
|
56
|
+
from zenml.integrations.pigeon import PigeonIntegration # noqa
|
56
57
|
from zenml.integrations.pillow import PillowIntegration # noqa
|
57
58
|
from zenml.integrations.polars import PolarsIntegration # noqa
|
58
59
|
from zenml.integrations.prodigy import ProdigyIntegration # noqa
|
@@ -78,3 +79,4 @@ from zenml.integrations.tensorflow import TensorflowIntegration # noqa
|
|
78
79
|
from zenml.integrations.wandb import WandbIntegration # noqa
|
79
80
|
from zenml.integrations.whylogs import WhylogsIntegration # noqa
|
80
81
|
from zenml.integrations.xgboost import XgboostIntegration # noqa
|
82
|
+
from zenml.integrations.vllm import VLLMIntegration # noqa
|
@@ -478,7 +478,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
478
478
|
)
|
479
479
|
|
480
480
|
# Yield metadata based on the generated execution object
|
481
|
-
yield from self.compute_metadata(
|
481
|
+
yield from self.compute_metadata(
|
482
|
+
execution=execution, settings=settings
|
483
|
+
)
|
482
484
|
|
483
485
|
# mainly for testing purposes, we wait for the pipeline to finish
|
484
486
|
if settings.synchronous:
|
@@ -518,19 +520,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
518
520
|
"pipeline_execution_arn": pipeline_execution_arn,
|
519
521
|
}
|
520
522
|
|
521
|
-
aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
|
522
|
-
|
523
|
-
region_name, _, _ = dissect_pipeline_execution_arn(
|
524
|
-
pipeline_execution_arn=pipeline_execution_arn
|
525
|
-
)
|
526
|
-
|
527
|
-
orchestrator_logs_url = (
|
528
|
-
f"https://{region_name}.console.aws.amazon.com/"
|
529
|
-
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
530
|
-
f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
|
531
|
-
f"$3Dpipelines-{aws_run_id}-"
|
532
|
-
)
|
533
|
-
run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
|
534
523
|
return run_metadata
|
535
524
|
|
536
525
|
def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
|
@@ -566,7 +555,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
566
555
|
|
567
556
|
# Fetch the status of the _PipelineExecution
|
568
557
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
569
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
558
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
570
559
|
elif run.orchestrator_run_id is not None:
|
571
560
|
run_id = run.orchestrator_run_id
|
572
561
|
else:
|
@@ -590,12 +579,15 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
590
579
|
raise ValueError("Unknown status for the pipeline execution.")
|
591
580
|
|
592
581
|
def compute_metadata(
|
593
|
-
self,
|
582
|
+
self,
|
583
|
+
execution: Any,
|
584
|
+
settings: SagemakerOrchestratorSettings,
|
594
585
|
) -> Iterator[Dict[str, MetadataType]]:
|
595
586
|
"""Generate run metadata based on the generated Sagemaker Execution.
|
596
587
|
|
597
588
|
Args:
|
598
589
|
execution: The corresponding _PipelineExecution object.
|
590
|
+
settings: The Sagemaker orchestrator settings.
|
599
591
|
|
600
592
|
Yields:
|
601
593
|
A dictionary of metadata related to the pipeline run.
|
@@ -612,7 +604,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
612
604
|
metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_url)
|
613
605
|
|
614
606
|
# URL to the corresponding CloudWatch page
|
615
|
-
if logs_url := self._compute_orchestrator_logs_url(
|
607
|
+
if logs_url := self._compute_orchestrator_logs_url(
|
608
|
+
execution, settings
|
609
|
+
):
|
616
610
|
metadata[METADATA_ORCHESTRATOR_LOGS_URL] = Uri(logs_url)
|
617
611
|
|
618
612
|
yield metadata
|
@@ -656,11 +650,13 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
656
650
|
@staticmethod
|
657
651
|
def _compute_orchestrator_logs_url(
|
658
652
|
pipeline_execution: Any,
|
653
|
+
settings: SagemakerOrchestratorSettings,
|
659
654
|
) -> Optional[str]:
|
660
655
|
"""Generate the CloudWatch URL upon pipeline execution.
|
661
656
|
|
662
657
|
Args:
|
663
658
|
pipeline_execution: The corresponding _PipelineExecution object.
|
659
|
+
settings: The Sagemaker orchestrator settings.
|
664
660
|
|
665
661
|
Returns:
|
666
662
|
the URL querying the pipeline logs in CloudWatch on AWS.
|
@@ -670,10 +666,16 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
670
666
|
pipeline_execution.arn
|
671
667
|
)
|
672
668
|
|
669
|
+
use_training_jobs = True
|
670
|
+
if settings.use_training_step is not None:
|
671
|
+
use_training_jobs = settings.use_training_step
|
672
|
+
|
673
|
+
job_type = "Training" if use_training_jobs else "Processing"
|
674
|
+
|
673
675
|
return (
|
674
676
|
f"https://{region_name}.console.aws.amazon.com/"
|
675
677
|
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
676
|
-
f"/$252Faws$252Fsagemaker$
|
678
|
+
f"/$252Faws$252Fsagemaker$252F{job_type}Jobs$3FlogStreamNameFilter"
|
677
679
|
f"$3Dpipelines-{execution_id}-"
|
678
680
|
)
|
679
681
|
except Exception as e:
|
@@ -482,7 +482,7 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
|
|
482
482
|
|
483
483
|
# Fetch the status of the PipelineJob
|
484
484
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
485
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
485
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
486
486
|
elif run.orchestrator_run_id is not None:
|
487
487
|
run_id = run.orchestrator_run_id
|
488
488
|
else:
|
@@ -14,7 +14,6 @@
|
|
14
14
|
"""Materializer for BentoML Bento objects."""
|
15
15
|
|
16
16
|
import os
|
17
|
-
import tempfile
|
18
17
|
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type
|
19
18
|
|
20
19
|
import bentoml
|
@@ -23,7 +22,6 @@ from bentoml.exceptions import BentoMLException
|
|
23
22
|
|
24
23
|
from zenml.enums import ArtifactType
|
25
24
|
from zenml.integrations.bentoml.constants import DEFAULT_BENTO_FILENAME
|
26
|
-
from zenml.io import fileio
|
27
25
|
from zenml.logger import get_logger
|
28
26
|
from zenml.materializers.base_materializer import BaseMaterializer
|
29
27
|
from zenml.utils import io_utils
|
@@ -49,23 +47,21 @@ class BentoMaterializer(BaseMaterializer):
|
|
49
47
|
Returns:
|
50
48
|
An bento.Bento object.
|
51
49
|
"""
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
imported_bento.save()
|
68
|
-
return imported_bento
|
50
|
+
with self.get_temporary_directory(delete_at_exit=False) as temp_dir:
|
51
|
+
# Copy from artifact store to temporary directory
|
52
|
+
io_utils.copy_dir(self.uri, temp_dir)
|
53
|
+
|
54
|
+
# Load the Bento from the temporary directory
|
55
|
+
imported_bento = Bento.import_from(
|
56
|
+
os.path.join(temp_dir, DEFAULT_BENTO_FILENAME)
|
57
|
+
)
|
58
|
+
|
59
|
+
# Try save the Bento to the local BentoML store
|
60
|
+
try:
|
61
|
+
_ = bentoml.get(imported_bento.tag)
|
62
|
+
except BentoMLException:
|
63
|
+
imported_bento.save()
|
64
|
+
return imported_bento
|
69
65
|
|
70
66
|
def save(self, bento: bento.Bento) -> None:
|
71
67
|
"""Write to artifact store.
|
@@ -73,18 +69,10 @@ class BentoMaterializer(BaseMaterializer):
|
|
73
69
|
Args:
|
74
70
|
bento: An bento.Bento object.
|
75
71
|
"""
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
# save the image in a temporary directory
|
81
|
-
bentoml.export_bento(bento.tag, temp_bento_path)
|
82
|
-
|
83
|
-
# copy the saved image to the artifact store
|
84
|
-
io_utils.copy_dir(temp_dir.name, self.uri)
|
85
|
-
|
86
|
-
# Remove the temporary directory
|
87
|
-
fileio.rmtree(temp_dir.name)
|
72
|
+
with self.get_temporary_directory(delete_at_exit=True) as temp_dir:
|
73
|
+
temp_bento_path = os.path.join(temp_dir, DEFAULT_BENTO_FILENAME)
|
74
|
+
bentoml.export_bento(bento.tag, temp_bento_path)
|
75
|
+
io_utils.copy_dir(temp_dir, self.uri)
|
88
76
|
|
89
77
|
def extract_metadata(
|
90
78
|
self, bento: bento.Bento
|
zenml/integrations/constants.py
CHANGED
@@ -430,7 +430,7 @@ class DeepchecksDataValidator(BaseDataValidator):
|
|
430
430
|
"""Run one or more Deepchecks model validation checks.
|
431
431
|
|
432
432
|
Call this method to perform model validation checks (e.g. confusion
|
433
|
-
matrix validation, performance reports, model error
|
433
|
+
matrix validation, performance reports, model error analyzes, etc).
|
434
434
|
A second dataset is required for model performance comparison tests
|
435
435
|
(i.e. tests that identify changes in a model behavior by comparing how
|
436
436
|
it performs on two different datasets).
|
@@ -33,7 +33,7 @@ from zenml.stack import Flavor
|
|
33
33
|
|
34
34
|
# Fix numba errors in Docker and suppress logs and deprecation warning spam
|
35
35
|
try:
|
36
|
-
from numba.core.errors import (
|
36
|
+
from numba.core.errors import (
|
37
37
|
NumbaDeprecationWarning,
|
38
38
|
NumbaPendingDeprecationWarning,
|
39
39
|
)
|
@@ -835,7 +835,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
835
835
|
|
836
836
|
# Fetch the status of the PipelineJob
|
837
837
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
838
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
838
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
839
839
|
elif run.orchestrator_run_id is not None:
|
840
840
|
run_id = run.orchestrator_run_id
|
841
841
|
else:
|
@@ -15,7 +15,6 @@
|
|
15
15
|
|
16
16
|
import os
|
17
17
|
from collections import defaultdict
|
18
|
-
from tempfile import TemporaryDirectory, mkdtemp
|
19
18
|
from typing import (
|
20
19
|
TYPE_CHECKING,
|
21
20
|
Any,
|
@@ -88,12 +87,12 @@ class HFDatasetMaterializer(BaseMaterializer):
|
|
88
87
|
Returns:
|
89
88
|
The dataset read from the specified dir.
|
90
89
|
"""
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
90
|
+
with self.get_temporary_directory(delete_at_exit=False) as temp_dir:
|
91
|
+
io_utils.copy_dir(
|
92
|
+
os.path.join(self.uri, DEFAULT_DATASET_DIR),
|
93
|
+
temp_dir,
|
94
|
+
)
|
95
|
+
return load_from_disk(temp_dir)
|
97
96
|
|
98
97
|
def save(self, ds: Union[Dataset, DatasetDict]) -> None:
|
99
98
|
"""Writes a Dataset to the specified dir.
|
@@ -101,16 +100,13 @@ class HFDatasetMaterializer(BaseMaterializer):
|
|
101
100
|
Args:
|
102
101
|
ds: The Dataset to write.
|
103
102
|
"""
|
104
|
-
|
105
|
-
|
106
|
-
try:
|
103
|
+
with self.get_temporary_directory(delete_at_exit=True) as temp_dir:
|
104
|
+
path = os.path.join(temp_dir, DEFAULT_DATASET_DIR)
|
107
105
|
ds.save_to_disk(path)
|
108
106
|
io_utils.copy_dir(
|
109
107
|
path,
|
110
108
|
os.path.join(self.uri, DEFAULT_DATASET_DIR),
|
111
109
|
)
|
112
|
-
finally:
|
113
|
-
fileio.rmtree(temp_dir.name)
|
114
110
|
|
115
111
|
def extract_metadata(
|
116
112
|
self, ds: Union[Dataset, DatasetDict]
|