zenml-nightly 0.68.1.dev20241103__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +77 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/artifacts/external_artifact.py +2 -1
- zenml/artifacts/{load_directory_materializer.py → preexisting_data_materializer.py} +8 -9
- zenml/artifacts/utils.py +139 -80
- zenml/cli/base.py +4 -4
- zenml/cli/model.py +1 -6
- zenml/cli/stack.py +1 -0
- zenml/client.py +29 -74
- zenml/config/server_config.py +17 -1
- zenml/constants.py +2 -7
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/enums.py +20 -4
- zenml/exceptions.py +4 -0
- zenml/integrations/__init__.py +3 -1
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
- zenml/integrations/bentoml/materializers/bentoml_bento_materializer.py +19 -31
- zenml/integrations/constants.py +1 -0
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
- zenml/integrations/evidently/__init__.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +8 -12
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +2 -5
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +17 -18
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -3
- zenml/integrations/langchain/__init__.py +2 -1
- zenml/integrations/langchain/materializers/openai_embedding_materializer.py +28 -2
- zenml/integrations/lightgbm/materializers/lightgbm_booster_materializer.py +8 -15
- zenml/integrations/lightgbm/materializers/lightgbm_dataset_materializer.py +11 -16
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
- zenml/integrations/openai/__init__.py +1 -1
- zenml/integrations/openai/hooks/open_ai_failure_hook.py +39 -14
- zenml/integrations/pillow/materializers/pillow_image_materializer.py +17 -20
- zenml/integrations/polars/materializers/dataframe_materializer.py +26 -39
- zenml/integrations/pycaret/materializers/model_materializer.py +7 -22
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
- zenml/integrations/tensorflow/materializers/keras_materializer.py +11 -22
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +8 -15
- zenml/integrations/vllm/__init__.py +50 -0
- zenml/integrations/vllm/flavors/__init__.py +21 -0
- zenml/integrations/vllm/flavors/vllm_model_deployer_flavor.py +91 -0
- zenml/integrations/vllm/model_deployers/__init__.py +19 -0
- zenml/integrations/vllm/model_deployers/vllm_model_deployer.py +263 -0
- zenml/integrations/vllm/services/__init__.py +19 -0
- zenml/integrations/vllm/services/vllm_deployment.py +206 -0
- zenml/integrations/whylogs/materializers/whylogs_materializer.py +11 -18
- zenml/integrations/xgboost/materializers/xgboost_booster_materializer.py +11 -22
- zenml/integrations/xgboost/materializers/xgboost_dmatrix_materializer.py +10 -19
- zenml/materializers/base_materializer.py +68 -1
- zenml/metadata/lazy_load.py +20 -7
- zenml/model/model.py +17 -64
- zenml/model/utils.py +5 -0
- zenml/models/__init__.py +0 -12
- zenml/models/v2/base/filter.py +121 -8
- zenml/models/v2/core/artifact_version.py +42 -7
- zenml/models/v2/core/model_version.py +26 -5
- zenml/models/v2/core/pipeline_run.py +25 -6
- zenml/models/v2/core/run_metadata.py +2 -217
- zenml/models/v2/core/step_run.py +62 -24
- zenml/orchestrators/base_orchestrator.py +12 -1
- zenml/orchestrators/input_utils.py +44 -19
- zenml/orchestrators/step_launcher.py +4 -3
- zenml/orchestrators/step_run_utils.py +19 -15
- zenml/orchestrators/step_runner.py +25 -14
- zenml/orchestrators/utils.py +45 -26
- zenml/stack/flavor.py +9 -5
- zenml/stack_deployments/aws_stack_deployment.py +23 -6
- zenml/stack_deployments/azure_stack_deployment.py +28 -5
- zenml/stack_deployments/gcp_stack_deployment.py +25 -8
- zenml/stack_deployments/stack_deployment.py +3 -5
- zenml/steps/base_step.py +1 -1
- zenml/steps/entrypoint_function_utils.py +3 -5
- zenml/steps/step_context.py +5 -2
- zenml/steps/utils.py +13 -2
- zenml/utils/callback_registry.py +71 -0
- zenml/utils/metadata_utils.py +335 -0
- zenml/zen_server/auth.py +221 -3
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
- zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
- zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
- zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
- zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
- zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
- zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
- zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
- zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
- zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
- zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
- zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
- zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
- zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
- zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
- zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
- zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
- zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
- zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
- zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
- zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
- zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
- zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
- zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +2 -0
- zenml/zen_server/jwt.py +30 -13
- zenml/zen_server/rbac/endpoint_utils.py +43 -1
- zenml/zen_server/rbac/utils.py +0 -2
- zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
- zenml/zen_server/routers/auth_endpoints.py +134 -102
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/workspaces_endpoints.py +3 -4
- zenml/zen_server/template_execution/utils.py +14 -16
- zenml/zen_server/utils.py +27 -0
- zenml/zen_server/zen_server_api.py +6 -3
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/rest_zen_store.py +68 -64
- zenml/zen_stores/schemas/artifact_schemas.py +8 -1
- zenml/zen_stores/schemas/model_schemas.py +27 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
- zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
- zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
- zenml/zen_stores/schemas/step_run_schemas.py +18 -10
- zenml/zen_stores/sql_zen_store.py +283 -219
- zenml/zen_stores/zen_store_interface.py +15 -42
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +226 -203
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
- zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
- zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
- zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
- zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
- zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
- zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
- zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
- zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
- zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.68.1.dev20241103.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
README.md
CHANGED
@@ -333,7 +333,7 @@ the Apache License Version 2.0.
|
|
333
333
|
<a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
|
334
334
|
<br />
|
335
335
|
<br />
|
336
|
-
🎉 Version 0.
|
336
|
+
🎉 Version 0.70.0 is out. Check out the release notes
|
337
337
|
<a href="https://github.com/zenml-io/zenml/releases">here</a>.
|
338
338
|
<br />
|
339
339
|
🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.
|
RELEASE_NOTES.md
CHANGED
@@ -1,5 +1,82 @@
|
|
1
1
|
<!-- markdown-link-check-disable -->
|
2
2
|
|
3
|
+
# 0.70.0
|
4
|
+
|
5
|
+
The **ZenML 0.70.0** release includes a significant number of database schema changes and migrations, which means upgrading to this version will require extra caution. As always, please make sure to make a copy of your production database before upgrading.
|
6
|
+
|
7
|
+
## Key Changes
|
8
|
+
|
9
|
+
* **Artifact Versioning Improvements**: The handling of artifact versions has been improved, including the API improvements like the ability to batch artifact version requests to improve the execution times and more types for the step input/output artifacts, including multiple versions of the same artifact (e.g. model checkpoints), to improve the UX using ZenML UI or while working directly with the API.
|
10
|
+
* **Scalability Enhancements**: Various scalability improvements have been made, such as reducing unnecessary server requests and incrementing artifact versions server-side. These enhancements are expected to provide significant speed and scale improvements for ZenML users.
|
11
|
+
* **Metadata management**: Now, all the metadata-creating functions are gathered under one method called `log_metadata`. It is possible to call this method with different inputs to log run metadata for artifact versions, model versions, steps, and runs.
|
12
|
+
* **The oneof filtering**: This allows to filter entities using a new operator called `oneof`. You can use this with IDs (UUID type) or tags (or other string-typed attributes) like this `PipelineRunFilter(tag='oneof:["cats", "dogs"]')`.
|
13
|
+
* **Documentation Improvements**: The ZenML documentation has been restructured and expanded, including the addition of new sections on [finetuning](https://docs.zenml.io/user-guide/llmops-guide/finetuning-llms) and [LLM/ML engineering](https://docs.zenml.io/user-guide/llmops-guide/evaluation) resources.
|
14
|
+
* **Bug Fixes**: This release includes several bug fixes, including issues with in-process main module source loading, and more.
|
15
|
+
|
16
|
+
## Caution: Make sure to back up your data before upgrading!
|
17
|
+
While this release brings many valuable improvements, the database schema changes and migrations pose a potential risk to users. It is strongly recommended that users:
|
18
|
+
|
19
|
+
* **Test the upgrade on a non-production environment**: Before upgrading a production system, test the upgrade process in a non-production environment to identify and address any issues.
|
20
|
+
* **Back up your data**: Ensure that you have a reliable backup of your ZenML data before attempting the upgrade.
|
21
|
+
|
22
|
+
## What's Changed
|
23
|
+
* Optimizing the CI workflows by @bcdurak in https://github.com/zenml-io/zenml/pull/3145
|
24
|
+
* Adding 0.68.0 to the migration tests by @bcdurak in https://github.com/zenml-io/zenml/pull/3144
|
25
|
+
* Move step durations to body by @schustmi in https://github.com/zenml-io/zenml/pull/3046
|
26
|
+
* Docs on ZenML setup by @strickvl in https://github.com/zenml-io/zenml/pull/3100
|
27
|
+
* Remove wrongly set Model.was_created_in_this_run attribute by @schustmi in https://github.com/zenml-io/zenml/pull/3129
|
28
|
+
* Allow specifying run tags in pipeline configuration by @schustmi in https://github.com/zenml-io/zenml/pull/3130
|
29
|
+
* Fix materializer type compatibility check during loading by @schustmi in https://github.com/zenml-io/zenml/pull/3105
|
30
|
+
* [docs] Add icons to headers in docs by @wjayesh in https://github.com/zenml-io/zenml/pull/3149
|
31
|
+
* fix icons and remove redundant file by @wjayesh in https://github.com/zenml-io/zenml/pull/3150
|
32
|
+
* Merge 0.68.1 release into develop by @schustmi in https://github.com/zenml-io/zenml/pull/3153
|
33
|
+
* Allow filtering pipeline runs by stack component by @schustmi in https://github.com/zenml-io/zenml/pull/3142
|
34
|
+
* Allow artifact response as step input by @schustmi in https://github.com/zenml-io/zenml/pull/3134
|
35
|
+
* Filter component by user name by @schustmi in https://github.com/zenml-io/zenml/pull/3126
|
36
|
+
* [docs] Restructure how-to section to make it more readable by @wjayesh in https://github.com/zenml-io/zenml/pull/3147
|
37
|
+
* ZenML Pro web login implementation by @stefannica in https://github.com/zenml-io/zenml/pull/3141
|
38
|
+
* Scalability improvements: Reduce misc/hydration server requests by @schustmi in https://github.com/zenml-io/zenml/pull/3093
|
39
|
+
* Fix in-process main module source loading by @schustmi in https://github.com/zenml-io/zenml/pull/3119
|
40
|
+
* Catch assertion in GH library by @schustmi in https://github.com/zenml-io/zenml/pull/3160
|
41
|
+
* Enable cache precomputation for run templates by @schustmi in https://github.com/zenml-io/zenml/pull/3156
|
42
|
+
* Add LLM and ML engineering books to README by @htahir1 in https://github.com/zenml-io/zenml/pull/3159
|
43
|
+
* Add helper method to quickly create run template from pipeline by @schustmi in https://github.com/zenml-io/zenml/pull/3155
|
44
|
+
* Add CLI command to export stack requirements by @schustmi in https://github.com/zenml-io/zenml/pull/3158
|
45
|
+
* Scalability improvements: Increment artifact version server side by @schustmi in https://github.com/zenml-io/zenml/pull/3095
|
46
|
+
* Update OpenAI integration by @safoinme in https://github.com/zenml-io/zenml/pull/3163
|
47
|
+
* Remove deprecated torch version constraint by @safoinme in https://github.com/zenml-io/zenml/pull/3166
|
48
|
+
* vLLM model deployer by @dudeperf3ct in https://github.com/zenml-io/zenml/pull/3032
|
49
|
+
* Don't initialize client during flavor sync by @schustmi in https://github.com/zenml-io/zenml/pull/3168
|
50
|
+
* Cleanup materializer temporary directories after step execution by @schustmi in https://github.com/zenml-io/zenml/pull/3162
|
51
|
+
* Fix langchain in API docs by @avishniakov in https://github.com/zenml-io/zenml/pull/3171
|
52
|
+
* Finetuning guide by @strickvl in https://github.com/zenml-io/zenml/pull/3157
|
53
|
+
* Fix mypy issue vllm evidently by @safoinme in https://github.com/zenml-io/zenml/pull/3169
|
54
|
+
* Add artifact version batch request by @schustmi in https://github.com/zenml-io/zenml/pull/3164
|
55
|
+
* Add missing section links by @strickvl in https://github.com/zenml-io/zenml/pull/3172
|
56
|
+
* Fix uvloop mypy by @avishniakov in https://github.com/zenml-io/zenml/pull/3174
|
57
|
+
* Multiple output versions for a step outputs by @avishniakov in https://github.com/zenml-io/zenml/pull/3072
|
58
|
+
* Simplify Metadata handling by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3096
|
59
|
+
* assign value to component_name in preset stack registration by @hirekk in https://github.com/zenml-io/zenml/pull/3178
|
60
|
+
* Updating the template versions with `zenml login` by @bcdurak in https://github.com/zenml-io/zenml/pull/3177
|
61
|
+
* Better input artifacts typing by @avishniakov in https://github.com/zenml-io/zenml/pull/3099
|
62
|
+
* Refactor environment setup and caching by @safoinme in https://github.com/zenml-io/zenml/pull/3077
|
63
|
+
* Fix spelling errors by @safoinme in https://github.com/zenml-io/zenml/pull/3181
|
64
|
+
* Prevent some race conditions by @schustmi in https://github.com/zenml-io/zenml/pull/3167
|
65
|
+
* Update stack deployments with latest features by @stefannica in https://github.com/zenml-io/zenml/pull/3183
|
66
|
+
* Terraform best practices by @htahir1 in https://github.com/zenml-io/zenml/pull/3131
|
67
|
+
* Fix sagemaker pipeline URLs by @stefannica in https://github.com/zenml-io/zenml/pull/3176
|
68
|
+
* Fix lightning orchestrator for multi-step pipelines by @wjayesh in https://github.com/zenml-io/zenml/pull/3170
|
69
|
+
* Port bugfixes from #2497 by @avishniakov in https://github.com/zenml-io/zenml/pull/3179
|
70
|
+
* Removing the `enable_cache` from the config files by @bcdurak in https://github.com/zenml-io/zenml/pull/3184
|
71
|
+
* Don't pass tags to step config by @schustmi in https://github.com/zenml-io/zenml/pull/3186
|
72
|
+
* New `log_metadata` function, new `oneof` filtering, additional `run_metadata` filtering by @bcdurak in https://github.com/zenml-io/zenml/pull/3182
|
73
|
+
|
74
|
+
## New Contributors
|
75
|
+
* @hirekk made their first contribution in https://github.com/zenml-io/zenml/pull/3178
|
76
|
+
|
77
|
+
**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.68.1...0.70.0
|
78
|
+
|
79
|
+
|
3
80
|
# 0.68.1
|
4
81
|
|
5
82
|
Fixes an issue with some partially cached pipelines running on remote orchestrators.
|
zenml/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.70.0.dev20241115
|
zenml/__init__.py
CHANGED
@@ -48,6 +48,7 @@ from zenml.model.model import Model
|
|
48
48
|
from zenml.pipelines import get_pipeline_context, pipeline
|
49
49
|
from zenml.steps import step, get_step_context
|
50
50
|
from zenml.steps.utils import log_step_metadata
|
51
|
+
from zenml.utils.metadata_utils import log_metadata
|
51
52
|
from zenml.entrypoints import entrypoint
|
52
53
|
|
53
54
|
__all__ = [
|
@@ -56,6 +57,7 @@ __all__ = [
|
|
56
57
|
"get_pipeline_context",
|
57
58
|
"get_step_context",
|
58
59
|
"load_artifact",
|
60
|
+
"log_metadata",
|
59
61
|
"log_artifact_metadata",
|
60
62
|
"log_model_metadata",
|
61
63
|
"log_step_metadata",
|
@@ -23,6 +23,7 @@ from zenml.artifacts.external_artifact_config import (
|
|
23
23
|
ExternalArtifactConfiguration,
|
24
24
|
)
|
25
25
|
from zenml.config.source import Source
|
26
|
+
from zenml.enums import ArtifactSaveType
|
26
27
|
from zenml.logger import get_logger
|
27
28
|
from zenml.materializers.base_materializer import BaseMaterializer
|
28
29
|
|
@@ -114,7 +115,7 @@ class ExternalArtifact(ExternalArtifactConfiguration):
|
|
114
115
|
materializer=self.materializer,
|
115
116
|
uri=uri,
|
116
117
|
has_custom_name=False,
|
117
|
-
|
118
|
+
save_type=ArtifactSaveType.EXTERNAL,
|
118
119
|
)
|
119
120
|
|
120
121
|
# To avoid duplicate uploads, switch to referencing the uploaded
|
@@ -14,7 +14,6 @@
|
|
14
14
|
"""Only-load materializer for directories."""
|
15
15
|
|
16
16
|
import os
|
17
|
-
import tempfile
|
18
17
|
from pathlib import Path
|
19
18
|
from typing import Any, ClassVar, Tuple, Type
|
20
19
|
|
@@ -46,14 +45,14 @@ class PreexistingDataMaterializer(BaseMaterializer):
|
|
46
45
|
Returns:
|
47
46
|
Path to the local directory that contains the artifact files.
|
48
47
|
"""
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
48
|
+
with self.get_temporary_directory(delete_at_exit=False) as temp_dir:
|
49
|
+
if fileio.isdir(self.uri):
|
50
|
+
self._copy_directory(src=self.uri, dst=temp_dir)
|
51
|
+
return Path(temp_dir)
|
52
|
+
else:
|
53
|
+
dst = os.path.join(temp_dir, os.path.split(self.uri)[-1])
|
54
|
+
fileio.copy(src=self.uri, dst=dst)
|
55
|
+
return Path(dst)
|
57
56
|
|
58
57
|
def save(self, data: Any) -> None:
|
59
58
|
"""Store the directory in the artifact store.
|
zenml/artifacts/utils.py
CHANGED
@@ -31,7 +31,7 @@ from typing import (
|
|
31
31
|
from uuid import UUID, uuid4
|
32
32
|
|
33
33
|
from zenml.artifacts.artifact_config import ArtifactConfig
|
34
|
-
from zenml.artifacts.
|
34
|
+
from zenml.artifacts.preexisting_data_materializer import (
|
35
35
|
PreexistingDataMaterializer,
|
36
36
|
)
|
37
37
|
from zenml.client import Client
|
@@ -39,6 +39,7 @@ from zenml.constants import (
|
|
39
39
|
MODEL_METADATA_YAML_FILE_NAME,
|
40
40
|
)
|
41
41
|
from zenml.enums import (
|
42
|
+
ArtifactSaveType,
|
42
43
|
ArtifactType,
|
43
44
|
ExecutionStatus,
|
44
45
|
MetadataResourceTypes,
|
@@ -82,6 +83,117 @@ logger = get_logger(__name__)
|
|
82
83
|
# ----------
|
83
84
|
|
84
85
|
|
86
|
+
def _save_artifact_visualizations(
|
87
|
+
data: Any, materializer: "BaseMaterializer"
|
88
|
+
) -> List[ArtifactVisualizationRequest]:
|
89
|
+
"""Save artifact visualizations.
|
90
|
+
|
91
|
+
Args:
|
92
|
+
data: The data for which to save the visualizations.
|
93
|
+
materializer: The materializer that should be used to generate and
|
94
|
+
save the visualizations.
|
95
|
+
|
96
|
+
Returns:
|
97
|
+
List of requests for the saved visualizations.
|
98
|
+
"""
|
99
|
+
try:
|
100
|
+
visualizations = materializer.save_visualizations(data)
|
101
|
+
except Exception as e:
|
102
|
+
logger.warning("Failed to save artifact visualizations: %s", e)
|
103
|
+
return []
|
104
|
+
|
105
|
+
return [
|
106
|
+
ArtifactVisualizationRequest(
|
107
|
+
type=type,
|
108
|
+
uri=uri,
|
109
|
+
)
|
110
|
+
for uri, type in visualizations.items()
|
111
|
+
]
|
112
|
+
|
113
|
+
|
114
|
+
def _store_artifact_data_and_prepare_request(
|
115
|
+
data: Any,
|
116
|
+
name: str,
|
117
|
+
uri: str,
|
118
|
+
materializer_class: Type["BaseMaterializer"],
|
119
|
+
save_type: ArtifactSaveType,
|
120
|
+
version: Optional[Union[int, str]] = None,
|
121
|
+
tags: Optional[List[str]] = None,
|
122
|
+
store_metadata: bool = True,
|
123
|
+
store_visualizations: bool = True,
|
124
|
+
has_custom_name: bool = True,
|
125
|
+
metadata: Optional[Dict[str, "MetadataType"]] = None,
|
126
|
+
) -> ArtifactVersionRequest:
|
127
|
+
"""Store artifact data and prepare a request to the server.
|
128
|
+
|
129
|
+
Args:
|
130
|
+
data: The artifact data.
|
131
|
+
name: The artifact name.
|
132
|
+
uri: The artifact URI.
|
133
|
+
materializer_class: The materializer class to use for storing the
|
134
|
+
artifact data.
|
135
|
+
save_type: Save type of the artifact version.
|
136
|
+
version: The artifact version.
|
137
|
+
tags: Tags for the artifact version.
|
138
|
+
store_metadata: Whether to store metadata for the artifact version.
|
139
|
+
store_visualizations: Whether to store visualizations for the artifact
|
140
|
+
version.
|
141
|
+
has_custom_name: Whether the artifact has a custom name.
|
142
|
+
metadata: Metadata to store for the artifact version. This will be
|
143
|
+
ignored if `store_metadata` is set to `False`.
|
144
|
+
|
145
|
+
Returns:
|
146
|
+
Artifact version request for the artifact data that was stored.
|
147
|
+
"""
|
148
|
+
artifact_store = Client().active_stack.artifact_store
|
149
|
+
artifact_store.makedirs(uri)
|
150
|
+
|
151
|
+
materializer = materializer_class(uri=uri, artifact_store=artifact_store)
|
152
|
+
materializer.uri = materializer.uri.replace("\\", "/")
|
153
|
+
|
154
|
+
data_type = type(data)
|
155
|
+
materializer.validate_save_type_compatibility(data_type)
|
156
|
+
materializer.save(data)
|
157
|
+
|
158
|
+
visualizations = (
|
159
|
+
_save_artifact_visualizations(data=data, materializer=materializer)
|
160
|
+
if store_visualizations
|
161
|
+
else None
|
162
|
+
)
|
163
|
+
|
164
|
+
combined_metadata: Dict[str, "MetadataType"] = {}
|
165
|
+
if store_metadata:
|
166
|
+
try:
|
167
|
+
combined_metadata = materializer.extract_full_metadata(data)
|
168
|
+
except Exception as e:
|
169
|
+
logger.warning("Failed to extract materializer metadata: %s", e)
|
170
|
+
|
171
|
+
# Update with user metadata to potentially overwrite values coming from
|
172
|
+
# the materializer
|
173
|
+
combined_metadata.update(metadata or {})
|
174
|
+
|
175
|
+
artifact_version_request = ArtifactVersionRequest(
|
176
|
+
artifact_name=name,
|
177
|
+
version=version,
|
178
|
+
tags=tags,
|
179
|
+
type=materializer.ASSOCIATED_ARTIFACT_TYPE,
|
180
|
+
uri=materializer.uri,
|
181
|
+
materializer=source_utils.resolve(materializer.__class__),
|
182
|
+
data_type=source_utils.resolve(data_type),
|
183
|
+
user=Client().active_user.id,
|
184
|
+
workspace=Client().active_workspace.id,
|
185
|
+
artifact_store_id=artifact_store.id,
|
186
|
+
visualizations=visualizations,
|
187
|
+
has_custom_name=has_custom_name,
|
188
|
+
save_type=save_type,
|
189
|
+
metadata=validate_metadata(combined_metadata)
|
190
|
+
if combined_metadata
|
191
|
+
else None,
|
192
|
+
)
|
193
|
+
|
194
|
+
return artifact_version_request
|
195
|
+
|
196
|
+
|
85
197
|
def save_artifact(
|
86
198
|
data: Any,
|
87
199
|
name: str,
|
@@ -89,13 +201,14 @@ def save_artifact(
|
|
89
201
|
tags: Optional[List[str]] = None,
|
90
202
|
extract_metadata: bool = True,
|
91
203
|
include_visualizations: bool = True,
|
92
|
-
has_custom_name: bool = True,
|
93
204
|
user_metadata: Optional[Dict[str, "MetadataType"]] = None,
|
94
205
|
materializer: Optional["MaterializerClassOrSource"] = None,
|
95
206
|
uri: Optional[str] = None,
|
96
207
|
is_model_artifact: bool = False,
|
97
208
|
is_deployment_artifact: bool = False,
|
98
|
-
|
209
|
+
# TODO: remove these once external artifact does not use this function anymore
|
210
|
+
save_type: ArtifactSaveType = ArtifactSaveType.MANUAL,
|
211
|
+
has_custom_name: bool = True,
|
99
212
|
) -> "ArtifactVersionResponse":
|
100
213
|
"""Upload and publish an artifact.
|
101
214
|
|
@@ -107,8 +220,6 @@ def save_artifact(
|
|
107
220
|
tags: Tags to associate with the artifact.
|
108
221
|
extract_metadata: If artifact metadata should be extracted and returned.
|
109
222
|
include_visualizations: If artifact visualizations should be generated.
|
110
|
-
has_custom_name: If the artifact name is custom and should be listed in
|
111
|
-
the dashboard "Artifacts" tab.
|
112
223
|
user_metadata: User-provided metadata to store with the artifact.
|
113
224
|
materializer: The materializer to use for saving the artifact to the
|
114
225
|
artifact store.
|
@@ -117,8 +228,9 @@ def save_artifact(
|
|
117
228
|
`custom_artifacts/{name}/{version}`.
|
118
229
|
is_model_artifact: If the artifact is a model artifact.
|
119
230
|
is_deployment_artifact: If the artifact is a deployment artifact.
|
120
|
-
|
121
|
-
|
231
|
+
save_type: The type of save operation that created the artifact version.
|
232
|
+
has_custom_name: If the artifact name is custom and should be listed in
|
233
|
+
the dashboard "Artifacts" tab.
|
122
234
|
|
123
235
|
Returns:
|
124
236
|
The saved artifact response.
|
@@ -129,17 +241,14 @@ def save_artifact(
|
|
129
241
|
from zenml.utils import source_utils
|
130
242
|
|
131
243
|
client = Client()
|
132
|
-
|
133
|
-
# Get the current artifact store
|
134
244
|
artifact_store = client.active_stack.artifact_store
|
135
245
|
|
136
|
-
# Build and check the artifact URI
|
137
246
|
if not uri:
|
138
247
|
uri = os.path.join("custom_artifacts", name, str(uuid4()))
|
139
248
|
if not uri.startswith(artifact_store.path):
|
140
249
|
uri = os.path.join(artifact_store.path, uri)
|
141
250
|
|
142
|
-
if
|
251
|
+
if save_type == ArtifactSaveType.MANUAL:
|
143
252
|
# This check is only necessary for manual saves as we already check
|
144
253
|
# it when creating the directory for step output artifacts
|
145
254
|
_check_if_artifact_with_given_uri_already_registered(
|
@@ -147,9 +256,7 @@ def save_artifact(
|
|
147
256
|
uri=uri,
|
148
257
|
name=name,
|
149
258
|
)
|
150
|
-
artifact_store.makedirs(uri)
|
151
259
|
|
152
|
-
# Find and initialize the right materializer class
|
153
260
|
if isinstance(materializer, type):
|
154
261
|
materializer_class = materializer
|
155
262
|
elif materializer:
|
@@ -158,66 +265,25 @@ def save_artifact(
|
|
158
265
|
)
|
159
266
|
else:
|
160
267
|
materializer_class = materializer_registry[type(data)]
|
161
|
-
materializer_object = materializer_class(uri)
|
162
|
-
|
163
|
-
# Force URIs to have forward slashes
|
164
|
-
materializer_object.uri = materializer_object.uri.replace("\\", "/")
|
165
|
-
|
166
|
-
# Save the artifact to the artifact store
|
167
|
-
data_type = type(data)
|
168
|
-
materializer_object.validate_save_type_compatibility(data_type)
|
169
|
-
materializer_object.save(data)
|
170
268
|
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
vis_model = ArtifactVisualizationRequest(
|
178
|
-
type=vis_type,
|
179
|
-
uri=vis_uri,
|
180
|
-
)
|
181
|
-
visualizations.append(vis_model)
|
182
|
-
except Exception as e:
|
183
|
-
logger.warning(
|
184
|
-
f"Failed to save visualization for output artifact '{name}': "
|
185
|
-
f"{e}"
|
186
|
-
)
|
187
|
-
|
188
|
-
# Save metadata of the artifact
|
189
|
-
artifact_metadata: Dict[str, "MetadataType"] = {}
|
190
|
-
if extract_metadata:
|
191
|
-
try:
|
192
|
-
artifact_metadata = materializer_object.extract_full_metadata(data)
|
193
|
-
artifact_metadata.update(user_metadata or {})
|
194
|
-
except Exception as e:
|
195
|
-
logger.warning(
|
196
|
-
f"Failed to extract metadata for output artifact '{name}': {e}"
|
197
|
-
)
|
198
|
-
|
199
|
-
artifact_version_request = ArtifactVersionRequest(
|
200
|
-
artifact_name=name,
|
269
|
+
artifact_version_request = _store_artifact_data_and_prepare_request(
|
270
|
+
data=data,
|
271
|
+
name=name,
|
272
|
+
uri=uri,
|
273
|
+
materializer_class=materializer_class,
|
274
|
+
save_type=save_type,
|
201
275
|
version=version,
|
202
276
|
tags=tags,
|
203
|
-
|
204
|
-
|
205
|
-
materializer=source_utils.resolve(materializer_object.__class__),
|
206
|
-
data_type=source_utils.resolve(data_type),
|
207
|
-
user=Client().active_user.id,
|
208
|
-
workspace=Client().active_workspace.id,
|
209
|
-
artifact_store_id=artifact_store.id,
|
210
|
-
visualizations=visualizations,
|
277
|
+
store_metadata=extract_metadata,
|
278
|
+
store_visualizations=include_visualizations,
|
211
279
|
has_custom_name=has_custom_name,
|
212
|
-
metadata=
|
213
|
-
if artifact_metadata
|
214
|
-
else None,
|
280
|
+
metadata=user_metadata,
|
215
281
|
)
|
216
282
|
artifact_version = client.zen_store.create_artifact_version(
|
217
283
|
artifact_version=artifact_version_request
|
218
284
|
)
|
219
285
|
|
220
|
-
if
|
286
|
+
if save_type == ArtifactSaveType.MANUAL:
|
221
287
|
_link_artifact_version_to_the_step_and_model(
|
222
288
|
artifact_version=artifact_version,
|
223
289
|
is_model_artifact=is_model_artifact,
|
@@ -281,6 +347,7 @@ def register_artifact(
|
|
281
347
|
version=version,
|
282
348
|
tags=tags,
|
283
349
|
type=ArtifactType.DATA,
|
350
|
+
save_type=ArtifactSaveType.PREEXISTING,
|
284
351
|
uri=folder_or_file_uri,
|
285
352
|
materializer=source_utils.resolve(PreexistingDataMaterializer),
|
286
353
|
data_type=source_utils.resolve(Path),
|
@@ -320,17 +387,6 @@ def load_artifact(
|
|
320
387
|
The loaded artifact.
|
321
388
|
"""
|
322
389
|
artifact = Client().get_artifact_version(name_or_id, version)
|
323
|
-
try:
|
324
|
-
step_run = get_step_context().step_run
|
325
|
-
client = Client()
|
326
|
-
client.zen_store.update_run_step(
|
327
|
-
step_run_id=step_run.id,
|
328
|
-
step_run_update=StepRunUpdate(
|
329
|
-
loaded_artifact_versions={artifact.name: artifact.id}
|
330
|
-
),
|
331
|
-
)
|
332
|
-
except RuntimeError:
|
333
|
-
pass # Cannot link to step run if called outside of a step
|
334
390
|
return load_artifact_from_response(artifact)
|
335
391
|
|
336
392
|
|
@@ -352,7 +408,7 @@ def log_artifact_metadata(
|
|
352
408
|
not provided, when being called inside a step that produces an
|
353
409
|
artifact named `artifact_name`, the metadata will be associated to
|
354
410
|
the corresponding newly created artifact. Or, if not provided when
|
355
|
-
being called outside
|
411
|
+
being called outside a step, or in a step that does not produce
|
356
412
|
any artifact named `artifact_name`, the metadata will be associated
|
357
413
|
to the latest version of that artifact.
|
358
414
|
|
@@ -361,6 +417,10 @@ def log_artifact_metadata(
|
|
361
417
|
called inside a step with a single output, or, if neither an
|
362
418
|
artifact nor an output with the given name exists.
|
363
419
|
"""
|
420
|
+
logger.warning(
|
421
|
+
"The `log_artifact_metadata` function is deprecated and will soon be "
|
422
|
+
"removed. Please use `log_metadata` instead."
|
423
|
+
)
|
364
424
|
try:
|
365
425
|
step_context = get_step_context()
|
366
426
|
in_step_outputs = (artifact_name in step_context._outputs) or (
|
@@ -574,7 +634,8 @@ def get_artifacts_versions_of_pipeline_run(
|
|
574
634
|
artifact_versions: List["ArtifactVersionResponse"] = []
|
575
635
|
for step in pipeline_run.steps.values():
|
576
636
|
if not only_produced or step.status == ExecutionStatus.COMPLETED:
|
577
|
-
|
637
|
+
for output in step.outputs.values():
|
638
|
+
artifact_versions.extend(output)
|
578
639
|
return artifact_versions
|
579
640
|
|
580
641
|
|
@@ -635,9 +696,7 @@ def _link_artifact_version_to_the_step_and_model(
|
|
635
696
|
client.zen_store.update_run_step(
|
636
697
|
step_run_id=step_run.id,
|
637
698
|
step_run_update=StepRunUpdate(
|
638
|
-
|
639
|
-
artifact_version.artifact.name: artifact_version.id
|
640
|
-
}
|
699
|
+
outputs={artifact_version.artifact.name: artifact_version.id}
|
641
700
|
),
|
642
701
|
)
|
643
702
|
error_message = "model"
|
zenml/cli/base.py
CHANGED
@@ -79,19 +79,19 @@ class ZenMLProjectTemplateLocation(BaseModel):
|
|
79
79
|
ZENML_PROJECT_TEMPLATES = dict(
|
80
80
|
e2e_batch=ZenMLProjectTemplateLocation(
|
81
81
|
github_url="zenml-io/template-e2e-batch",
|
82
|
-
github_tag="2024.10.
|
82
|
+
github_tag="2024.10.30", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
83
83
|
),
|
84
84
|
starter=ZenMLProjectTemplateLocation(
|
85
85
|
github_url="zenml-io/template-starter",
|
86
|
-
github_tag="2024.
|
86
|
+
github_tag="2024.10.30", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
87
87
|
),
|
88
88
|
nlp=ZenMLProjectTemplateLocation(
|
89
89
|
github_url="zenml-io/template-nlp",
|
90
|
-
github_tag="2024.
|
90
|
+
github_tag="2024.10.30", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
91
91
|
),
|
92
92
|
llm_finetuning=ZenMLProjectTemplateLocation(
|
93
93
|
github_url="zenml-io/template-llm-finetuning",
|
94
|
-
github_tag="2024.
|
94
|
+
github_tag="2024.11.08", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
95
95
|
),
|
96
96
|
)
|
97
97
|
|
zenml/cli/model.py
CHANGED
@@ -59,11 +59,6 @@ def _model_to_print(model: ModelResponse) -> Dict[str, Any]:
|
|
59
59
|
def _model_version_to_print(
|
60
60
|
model_version: ModelVersionResponse,
|
61
61
|
) -> Dict[str, Any]:
|
62
|
-
run_metadata = None
|
63
|
-
if model_version.run_metadata:
|
64
|
-
run_metadata = {
|
65
|
-
k: v.value for k, v in model_version.run_metadata.items()
|
66
|
-
}
|
67
62
|
return {
|
68
63
|
"id": model_version.id,
|
69
64
|
"model": model_version.model.name,
|
@@ -71,7 +66,7 @@ def _model_version_to_print(
|
|
71
66
|
"number": model_version.number,
|
72
67
|
"description": model_version.description,
|
73
68
|
"stage": model_version.stage,
|
74
|
-
"run_metadata": run_metadata,
|
69
|
+
"run_metadata": model_version.run_metadata,
|
75
70
|
"tags": [t.name for t in model_version.tags],
|
76
71
|
"data_artifacts_count": len(model_version.data_artifact_ids),
|
77
72
|
"model_artifacts_count": len(model_version.model_artifact_ids),
|
zenml/cli/stack.py
CHANGED