zenml-nightly 0.68.1.dev20241111__py3-none-any.whl → 0.70.0.dev20241114__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +77 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/artifacts/utils.py +5 -1
- zenml/cli/base.py +1 -1
- zenml/client.py +9 -2
- zenml/config/server_config.py +17 -1
- zenml/constants.py +1 -7
- zenml/enums.py +8 -0
- zenml/exceptions.py +4 -0
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +1 -14
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
- zenml/model/model.py +16 -62
- zenml/model/utils.py +5 -0
- zenml/models/v2/base/filter.py +121 -8
- zenml/models/v2/core/artifact_version.py +23 -0
- zenml/models/v2/core/model_version.py +23 -0
- zenml/models/v2/core/pipeline_run.py +22 -1
- zenml/models/v2/core/step_run.py +22 -0
- zenml/orchestrators/base_orchestrator.py +12 -1
- zenml/orchestrators/step_launcher.py +2 -1
- zenml/orchestrators/utils.py +45 -26
- zenml/stack_deployments/aws_stack_deployment.py +23 -6
- zenml/stack_deployments/azure_stack_deployment.py +28 -5
- zenml/stack_deployments/gcp_stack_deployment.py +25 -8
- zenml/stack_deployments/stack_deployment.py +3 -5
- zenml/steps/utils.py +5 -0
- zenml/utils/metadata_utils.py +335 -0
- zenml/zen_server/auth.py +221 -3
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
- zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
- zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
- zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
- zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
- zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
- zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
- zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
- zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
- zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
- zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
- zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
- zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
- zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
- zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
- zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
- zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
- zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
- zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
- zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
- zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
- zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
- zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
- zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +2 -0
- zenml/zen_server/jwt.py +30 -13
- zenml/zen_server/routers/auth_endpoints.py +134 -102
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/template_execution/utils.py +14 -16
- zenml/zen_server/utils.py +27 -0
- zenml/zen_server/zen_server_api.py +6 -1
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/rest_zen_store.py +13 -10
- zenml/zen_stores/schemas/model_schemas.py +25 -1
- zenml/zen_stores/schemas/pipeline_run_schemas.py +5 -0
- zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
- zenml/zen_stores/sql_zen_store.py +215 -121
- {zenml_nightly-0.68.1.dev20241111.dist-info → zenml_nightly-0.70.0.dev20241114.dist-info}/METADATA +2 -2
- {zenml_nightly-0.68.1.dev20241111.dist-info → zenml_nightly-0.70.0.dev20241114.dist-info}/RECORD +165 -151
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
- zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
- zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
- zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
- zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
- zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
- zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
- zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
- zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
- zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
- {zenml_nightly-0.68.1.dev20241111.dist-info → zenml_nightly-0.70.0.dev20241114.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.68.1.dev20241111.dist-info → zenml_nightly-0.70.0.dev20241114.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.68.1.dev20241111.dist-info → zenml_nightly-0.70.0.dev20241114.dist-info}/entry_points.txt +0 -0
README.md
CHANGED
@@ -333,7 +333,7 @@ the Apache License Version 2.0.
|
|
333
333
|
<a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
|
334
334
|
<br />
|
335
335
|
<br />
|
336
|
-
🎉 Version 0.
|
336
|
+
🎉 Version 0.70.0 is out. Check out the release notes
|
337
337
|
<a href="https://github.com/zenml-io/zenml/releases">here</a>.
|
338
338
|
<br />
|
339
339
|
🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.
|
RELEASE_NOTES.md
CHANGED
@@ -1,5 +1,82 @@
|
|
1
1
|
<!-- markdown-link-check-disable -->
|
2
2
|
|
3
|
+
# 0.70.0
|
4
|
+
|
5
|
+
The **ZenML 0.70.0** release includes a significant number of database schema changes and migrations, which means upgrading to this version will require extra caution. As always, please make sure to make a copy of your production database before upgrading.
|
6
|
+
|
7
|
+
## Key Changes
|
8
|
+
|
9
|
+
* **Artifact Versioning Improvements**: The handling of artifact versions has been improved, including the API improvements like the ability to batch artifact version requests to improve the execution times and more types for the step input/output artifacts, including multiple versions of the same artifact (e.g. model checkpoints), to improve the UX using ZenML UI or while working directly with the API.
|
10
|
+
* **Scalability Enhancements**: Various scalability improvements have been made, such as reducing unnecessary server requests and incrementing artifact versions server-side. These enhancements are expected to provide significant speed and scale improvements for ZenML users.
|
11
|
+
* **Metadata management**: Now, all the metadata-creating functions are gathered under one method called `log_metadata`. It is possible to call this method with different inputs to log run metadata for artifact versions, model versions, steps, and runs.
|
12
|
+
* **The oneof filtering**: This allows to filter entities using a new operator called `oneof`. You can use this with IDs (UUID type) or tags (or other string-typed attributes) like this `PipelineRunFilter(tag='oneof:["cats", "dogs"]')`.
|
13
|
+
* **Documentation Improvements**: The ZenML documentation has been restructured and expanded, including the addition of new sections on [finetuning](https://docs.zenml.io/user-guide/llmops-guide/finetuning-llms) and [LLM/ML engineering](https://docs.zenml.io/user-guide/llmops-guide/evaluation) resources.
|
14
|
+
* **Bug Fixes**: This release includes several bug fixes, including issues with in-process main module source loading, and more.
|
15
|
+
|
16
|
+
## Caution: Make sure to back up your data before upgrading!
|
17
|
+
While this release brings many valuable improvements, the database schema changes and migrations pose a potential risk to users. It is strongly recommended that users:
|
18
|
+
|
19
|
+
* **Test the upgrade on a non-production environment**: Before upgrading a production system, test the upgrade process in a non-production environment to identify and address any issues.
|
20
|
+
* **Back up your data**: Ensure that you have a reliable backup of your ZenML data before attempting the upgrade.
|
21
|
+
|
22
|
+
## What's Changed
|
23
|
+
* Optimizing the CI workflows by @bcdurak in https://github.com/zenml-io/zenml/pull/3145
|
24
|
+
* Adding 0.68.0 to the migration tests by @bcdurak in https://github.com/zenml-io/zenml/pull/3144
|
25
|
+
* Move step durations to body by @schustmi in https://github.com/zenml-io/zenml/pull/3046
|
26
|
+
* Docs on ZenML setup by @strickvl in https://github.com/zenml-io/zenml/pull/3100
|
27
|
+
* Remove wrongly set Model.was_created_in_this_run attribute by @schustmi in https://github.com/zenml-io/zenml/pull/3129
|
28
|
+
* Allow specifying run tags in pipeline configuration by @schustmi in https://github.com/zenml-io/zenml/pull/3130
|
29
|
+
* Fix materializer type compatibility check during loading by @schustmi in https://github.com/zenml-io/zenml/pull/3105
|
30
|
+
* [docs] Add icons to headers in docs by @wjayesh in https://github.com/zenml-io/zenml/pull/3149
|
31
|
+
* fix icons and remove redundant file by @wjayesh in https://github.com/zenml-io/zenml/pull/3150
|
32
|
+
* Merge 0.68.1 release into develop by @schustmi in https://github.com/zenml-io/zenml/pull/3153
|
33
|
+
* Allow filtering pipeline runs by stack component by @schustmi in https://github.com/zenml-io/zenml/pull/3142
|
34
|
+
* Allow artifact response as step input by @schustmi in https://github.com/zenml-io/zenml/pull/3134
|
35
|
+
* Filter component by user name by @schustmi in https://github.com/zenml-io/zenml/pull/3126
|
36
|
+
* [docs] Restructure how-to section to make it more readable by @wjayesh in https://github.com/zenml-io/zenml/pull/3147
|
37
|
+
* ZenML Pro web login implementation by @stefannica in https://github.com/zenml-io/zenml/pull/3141
|
38
|
+
* Scalability improvements: Reduce misc/hydration server requests by @schustmi in https://github.com/zenml-io/zenml/pull/3093
|
39
|
+
* Fix in-process main module source loading by @schustmi in https://github.com/zenml-io/zenml/pull/3119
|
40
|
+
* Catch assertion in GH library by @schustmi in https://github.com/zenml-io/zenml/pull/3160
|
41
|
+
* Enable cache precomputation for run templates by @schustmi in https://github.com/zenml-io/zenml/pull/3156
|
42
|
+
* Add LLM and ML engineering books to README by @htahir1 in https://github.com/zenml-io/zenml/pull/3159
|
43
|
+
* Add helper method to quickly create run template from pipeline by @schustmi in https://github.com/zenml-io/zenml/pull/3155
|
44
|
+
* Add CLI command to export stack requirements by @schustmi in https://github.com/zenml-io/zenml/pull/3158
|
45
|
+
* Scalability improvements: Increment artifact version server side by @schustmi in https://github.com/zenml-io/zenml/pull/3095
|
46
|
+
* Update OpenAI integration by @safoinme in https://github.com/zenml-io/zenml/pull/3163
|
47
|
+
* Remove deprecated torch version constraint by @safoinme in https://github.com/zenml-io/zenml/pull/3166
|
48
|
+
* vLLM model deployer by @dudeperf3ct in https://github.com/zenml-io/zenml/pull/3032
|
49
|
+
* Don't initialize client during flavor sync by @schustmi in https://github.com/zenml-io/zenml/pull/3168
|
50
|
+
* Cleanup materializer temporary directories after step execution by @schustmi in https://github.com/zenml-io/zenml/pull/3162
|
51
|
+
* Fix langchain in API docs by @avishniakov in https://github.com/zenml-io/zenml/pull/3171
|
52
|
+
* Finetuning guide by @strickvl in https://github.com/zenml-io/zenml/pull/3157
|
53
|
+
* Fix mypy issue vllm evidently by @safoinme in https://github.com/zenml-io/zenml/pull/3169
|
54
|
+
* Add artifact version batch request by @schustmi in https://github.com/zenml-io/zenml/pull/3164
|
55
|
+
* Add missing section links by @strickvl in https://github.com/zenml-io/zenml/pull/3172
|
56
|
+
* Fix uvloop mypy by @avishniakov in https://github.com/zenml-io/zenml/pull/3174
|
57
|
+
* Multiple output versions for a step outputs by @avishniakov in https://github.com/zenml-io/zenml/pull/3072
|
58
|
+
* Simplify Metadata handling by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3096
|
59
|
+
* assign value to component_name in preset stack registration by @hirekk in https://github.com/zenml-io/zenml/pull/3178
|
60
|
+
* Updating the template versions with `zenml login` by @bcdurak in https://github.com/zenml-io/zenml/pull/3177
|
61
|
+
* Better input artifacts typing by @avishniakov in https://github.com/zenml-io/zenml/pull/3099
|
62
|
+
* Refactor environment setup and caching by @safoinme in https://github.com/zenml-io/zenml/pull/3077
|
63
|
+
* Fix spelling errors by @safoinme in https://github.com/zenml-io/zenml/pull/3181
|
64
|
+
* Prevent some race conditions by @schustmi in https://github.com/zenml-io/zenml/pull/3167
|
65
|
+
* Update stack deployments with latest features by @stefannica in https://github.com/zenml-io/zenml/pull/3183
|
66
|
+
* Terraform best practices by @htahir1 in https://github.com/zenml-io/zenml/pull/3131
|
67
|
+
* Fix sagemaker pipeline URLs by @stefannica in https://github.com/zenml-io/zenml/pull/3176
|
68
|
+
* Fix lightning orchestrator for multi-step pipelines by @wjayesh in https://github.com/zenml-io/zenml/pull/3170
|
69
|
+
* Port bugfixes from #2497 by @avishniakov in https://github.com/zenml-io/zenml/pull/3179
|
70
|
+
* Removing the `enable_cache` from the config files by @bcdurak in https://github.com/zenml-io/zenml/pull/3184
|
71
|
+
* Don't pass tags to step config by @schustmi in https://github.com/zenml-io/zenml/pull/3186
|
72
|
+
* New `log_metadata` function, new `oneof` filtering, additional `run_metadata` filtering by @bcdurak in https://github.com/zenml-io/zenml/pull/3182
|
73
|
+
|
74
|
+
## New Contributors
|
75
|
+
* @hirekk made their first contribution in https://github.com/zenml-io/zenml/pull/3178
|
76
|
+
|
77
|
+
**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.68.1...0.70.0
|
78
|
+
|
79
|
+
|
3
80
|
# 0.68.1
|
4
81
|
|
5
82
|
Fixes an issue with some partially cached pipelines running on remote orchestrators.
|
zenml/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.70.0.dev20241114
|
zenml/__init__.py
CHANGED
@@ -48,6 +48,7 @@ from zenml.model.model import Model
|
|
48
48
|
from zenml.pipelines import get_pipeline_context, pipeline
|
49
49
|
from zenml.steps import step, get_step_context
|
50
50
|
from zenml.steps.utils import log_step_metadata
|
51
|
+
from zenml.utils.metadata_utils import log_metadata
|
51
52
|
from zenml.entrypoints import entrypoint
|
52
53
|
|
53
54
|
__all__ = [
|
@@ -56,6 +57,7 @@ __all__ = [
|
|
56
57
|
"get_pipeline_context",
|
57
58
|
"get_step_context",
|
58
59
|
"load_artifact",
|
60
|
+
"log_metadata",
|
59
61
|
"log_artifact_metadata",
|
60
62
|
"log_model_metadata",
|
61
63
|
"log_step_metadata",
|
zenml/artifacts/utils.py
CHANGED
@@ -408,7 +408,7 @@ def log_artifact_metadata(
|
|
408
408
|
not provided, when being called inside a step that produces an
|
409
409
|
artifact named `artifact_name`, the metadata will be associated to
|
410
410
|
the corresponding newly created artifact. Or, if not provided when
|
411
|
-
being called outside
|
411
|
+
being called outside a step, or in a step that does not produce
|
412
412
|
any artifact named `artifact_name`, the metadata will be associated
|
413
413
|
to the latest version of that artifact.
|
414
414
|
|
@@ -417,6 +417,10 @@ def log_artifact_metadata(
|
|
417
417
|
called inside a step with a single output, or, if neither an
|
418
418
|
artifact nor an output with the given name exists.
|
419
419
|
"""
|
420
|
+
logger.warning(
|
421
|
+
"The `log_artifact_metadata` function is deprecated and will soon be "
|
422
|
+
"removed. Please use `log_metadata` instead."
|
423
|
+
)
|
420
424
|
try:
|
421
425
|
step_context = get_step_context()
|
422
426
|
in_step_outputs = (artifact_name in step_context._outputs) or (
|
zenml/cli/base.py
CHANGED
@@ -91,7 +91,7 @@ ZENML_PROJECT_TEMPLATES = dict(
|
|
91
91
|
),
|
92
92
|
llm_finetuning=ZenMLProjectTemplateLocation(
|
93
93
|
github_url="zenml-io/template-llm-finetuning",
|
94
|
-
github_tag="2024.
|
94
|
+
github_tag="2024.11.08", # Make sure it is aligned with .github/workflows/update-templates-to-examples.yml
|
95
95
|
),
|
96
96
|
)
|
97
97
|
|
zenml/client.py
CHANGED
@@ -3426,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
|
|
3426
3426
|
Args:
|
3427
3427
|
id_or_prefix: The id or id prefix of the deployment.
|
3428
3428
|
"""
|
3429
|
-
deployment = self.get_deployment(
|
3429
|
+
deployment = self.get_deployment(
|
3430
|
+
id_or_prefix=id_or_prefix, hydrate=False
|
3431
|
+
)
|
3430
3432
|
self.zen_store.delete_deployment(deployment_id=deployment.id)
|
3431
3433
|
|
3432
3434
|
# ------------------------------ Run templates -----------------------------
|
@@ -3794,6 +3796,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3794
3796
|
templatable: Optional[bool] = None,
|
3795
3797
|
tag: Optional[str] = None,
|
3796
3798
|
user: Optional[Union[UUID, str]] = None,
|
3799
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
3797
3800
|
pipeline: Optional[Union[UUID, str]] = None,
|
3798
3801
|
code_repository: Optional[Union[UUID, str]] = None,
|
3799
3802
|
model: Optional[Union[UUID, str]] = None,
|
@@ -3833,6 +3836,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3833
3836
|
templatable: If the runs should be templatable or not.
|
3834
3837
|
tag: Tag to filter by.
|
3835
3838
|
user: The name/ID of the user to filter by.
|
3839
|
+
run_metadata: The run_metadata of the run to filter by.
|
3836
3840
|
pipeline: The name/ID of the pipeline to filter by.
|
3837
3841
|
code_repository: Filter by code repository name/ID.
|
3838
3842
|
model: Filter by model name/ID.
|
@@ -3872,6 +3876,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3872
3876
|
tag=tag,
|
3873
3877
|
unlisted=unlisted,
|
3874
3878
|
user=user,
|
3879
|
+
run_metadata=run_metadata,
|
3875
3880
|
pipeline=pipeline,
|
3876
3881
|
code_repository=code_repository,
|
3877
3882
|
stack=stack,
|
@@ -4192,7 +4197,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4192
4197
|
),
|
4193
4198
|
)
|
4194
4199
|
except RuntimeError:
|
4195
|
-
pass # Cannot link to step run if called outside
|
4200
|
+
pass # Cannot link to step run if called outside a step
|
4196
4201
|
return artifact
|
4197
4202
|
|
4198
4203
|
def list_artifact_versions(
|
@@ -4220,6 +4225,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4220
4225
|
user: Optional[Union[UUID, str]] = None,
|
4221
4226
|
model: Optional[Union[UUID, str]] = None,
|
4222
4227
|
pipeline_run: Optional[Union[UUID, str]] = None,
|
4228
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
4223
4229
|
tag: Optional[str] = None,
|
4224
4230
|
hydrate: bool = False,
|
4225
4231
|
) -> Page[ArtifactVersionResponse]:
|
@@ -4251,6 +4257,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4251
4257
|
user: Filter by user name or ID.
|
4252
4258
|
model: Filter by model name or ID.
|
4253
4259
|
pipeline_run: Filter by pipeline run name or ID.
|
4260
|
+
run_metadata: Filter by run metadata.
|
4254
4261
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
4255
4262
|
by including metadata fields in the response.
|
4256
4263
|
|
zenml/config/server_config.py
CHANGED
@@ -19,13 +19,14 @@ from secrets import token_hex
|
|
19
19
|
from typing import Any, Dict, List, Optional, Union
|
20
20
|
from uuid import UUID
|
21
21
|
|
22
|
-
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
22
|
+
from pydantic import BaseModel, ConfigDict, Field, PositiveInt, model_validator
|
23
23
|
|
24
24
|
from zenml.constants import (
|
25
25
|
DEFAULT_ZENML_JWT_TOKEN_ALGORITHM,
|
26
26
|
DEFAULT_ZENML_JWT_TOKEN_LEEWAY,
|
27
27
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING,
|
28
28
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT,
|
29
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME,
|
29
30
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY,
|
30
31
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE,
|
31
32
|
DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS,
|
@@ -119,6 +120,8 @@ class ServerConfiguration(BaseModel):
|
|
119
120
|
time of the JWT tokens issued to clients after they have
|
120
121
|
authenticated with the ZenML server using an OAuth 2.0 device
|
121
122
|
that has been marked as trusted.
|
123
|
+
generic_api_token_lifetime: The lifetime in seconds that generic
|
124
|
+
short-lived API tokens issued for automation purposes are valid.
|
122
125
|
external_login_url: The login URL of an external authenticator service
|
123
126
|
to use with the `EXTERNAL` authentication scheme.
|
124
127
|
external_user_info_url: The user info URL of an external authenticator
|
@@ -230,6 +233,12 @@ class ServerConfiguration(BaseModel):
|
|
230
233
|
deployment.
|
231
234
|
max_request_body_size_in_bytes: The maximum size of the request body in
|
232
235
|
bytes. If not specified, the default value of 256 Kb will be used.
|
236
|
+
memcache_max_capacity: The maximum number of entries that the memory
|
237
|
+
cache can hold. If not specified, the default value of 1000 will be
|
238
|
+
used.
|
239
|
+
memcache_default_expiry: The default expiry time in seconds for cache
|
240
|
+
entries. If not specified, the default value of 30 seconds will be
|
241
|
+
used.
|
233
242
|
"""
|
234
243
|
|
235
244
|
deployment_type: ServerDeploymentType = ServerDeploymentType.OTHER
|
@@ -257,6 +266,10 @@ class ServerConfiguration(BaseModel):
|
|
257
266
|
device_expiration_minutes: Optional[int] = None
|
258
267
|
trusted_device_expiration_minutes: Optional[int] = None
|
259
268
|
|
269
|
+
generic_api_token_lifetime: PositiveInt = (
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME
|
271
|
+
)
|
272
|
+
|
260
273
|
external_login_url: Optional[str] = None
|
261
274
|
external_user_info_url: Optional[str] = None
|
262
275
|
external_cookie_name: Optional[str] = None
|
@@ -321,6 +334,9 @@ class ServerConfiguration(BaseModel):
|
|
321
334
|
DEFAULT_ZENML_SERVER_MAX_REQUEST_BODY_SIZE_IN_BYTES
|
322
335
|
)
|
323
336
|
|
337
|
+
memcache_max_capacity: int = 1000
|
338
|
+
memcache_default_expiry: int = 30
|
339
|
+
|
324
340
|
_deployment_id: Optional[UUID] = None
|
325
341
|
|
326
342
|
@model_validator(mode="before")
|
zenml/constants.py
CHANGED
@@ -167,9 +167,6 @@ ENV_ZENML_SERVER = "ZENML_SERVER"
|
|
167
167
|
ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
|
168
168
|
ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS"
|
169
169
|
ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
|
170
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
|
171
|
-
"ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES"
|
172
|
-
)
|
173
170
|
ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
|
174
171
|
ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
|
175
172
|
ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
|
@@ -270,6 +267,7 @@ ZENML_API_KEY_PREFIX = "ZENKEY_"
|
|
270
267
|
DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48 # 48 hours
|
271
268
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
|
272
269
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY = 1000
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME = 60 * 60 # 1 hour
|
273
271
|
|
274
272
|
DEFAULT_ZENML_SERVER_SECURE_HEADERS_HSTS = (
|
275
273
|
"max-age=63072000; includeSubdomains"
|
@@ -410,10 +408,6 @@ MODEL_METADATA_YAML_FILE_NAME = "model_metadata.yaml"
|
|
410
408
|
|
411
409
|
# orchestrator constants
|
412
410
|
ORCHESTRATOR_DOCKER_IMAGE_KEY = "orchestrator"
|
413
|
-
PIPELINE_API_TOKEN_EXPIRES_MINUTES = handle_int_env_var(
|
414
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES,
|
415
|
-
default=60 * 24, # 24 hours
|
416
|
-
)
|
417
411
|
|
418
412
|
# Secret constants
|
419
413
|
SECRET_VALUES = "values"
|
zenml/enums.py
CHANGED
@@ -245,6 +245,13 @@ class OAuthDeviceStatus(StrEnum):
|
|
245
245
|
LOCKED = "locked"
|
246
246
|
|
247
247
|
|
248
|
+
class APITokenType(StrEnum):
|
249
|
+
"""The API token type."""
|
250
|
+
|
251
|
+
GENERIC = "generic"
|
252
|
+
WORKLOAD = "workload"
|
253
|
+
|
254
|
+
|
248
255
|
class GenericFilterOps(StrEnum):
|
249
256
|
"""Ops for all filters for string values on list methods."""
|
250
257
|
|
@@ -253,6 +260,7 @@ class GenericFilterOps(StrEnum):
|
|
253
260
|
CONTAINS = "contains"
|
254
261
|
STARTSWITH = "startswith"
|
255
262
|
ENDSWITH = "endswith"
|
263
|
+
ONEOF = "oneof"
|
256
264
|
GTE = "gte"
|
257
265
|
GT = "gt"
|
258
266
|
LTE = "lte"
|
zenml/exceptions.py
CHANGED
@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
|
|
168
168
|
"""Raised when trying to register an entity that already exists."""
|
169
169
|
|
170
170
|
|
171
|
+
class EntityCreationError(ZenMLBaseException, RuntimeError):
|
172
|
+
"""Raised when failing to create an entity."""
|
173
|
+
|
174
|
+
|
171
175
|
class ActionExistsError(EntityExistsError):
|
172
176
|
"""Raised when registering an action with a name that already exists."""
|
173
177
|
|
@@ -518,19 +518,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
518
518
|
"pipeline_execution_arn": pipeline_execution_arn,
|
519
519
|
}
|
520
520
|
|
521
|
-
aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
|
522
|
-
|
523
|
-
region_name, _, _ = dissect_pipeline_execution_arn(
|
524
|
-
pipeline_execution_arn=pipeline_execution_arn
|
525
|
-
)
|
526
|
-
|
527
|
-
orchestrator_logs_url = (
|
528
|
-
f"https://{region_name}.console.aws.amazon.com/"
|
529
|
-
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
530
|
-
f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
|
531
|
-
f"$3Dpipelines-{aws_run_id}-"
|
532
|
-
)
|
533
|
-
run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
|
534
521
|
return run_metadata
|
535
522
|
|
536
523
|
def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
|
@@ -673,7 +660,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
673
660
|
return (
|
674
661
|
f"https://{region_name}.console.aws.amazon.com/"
|
675
662
|
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
676
|
-
f"/$252Faws$252Fsagemaker$
|
663
|
+
f"/$252Faws$252Fsagemaker$252FTrainingJobs$3FlogStreamNameFilter"
|
677
664
|
f"$3Dpipelines-{execution_id}-"
|
678
665
|
)
|
679
666
|
except Exception as e:
|
@@ -390,10 +390,13 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
390
390
|
studio.run(
|
391
391
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
392
392
|
)
|
393
|
-
studio.upload_file(
|
394
|
-
|
395
|
-
|
393
|
+
studio.upload_file(env_file_path)
|
394
|
+
time.sleep(6)
|
395
|
+
studio.run(
|
396
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
396
397
|
)
|
398
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
399
|
+
|
397
400
|
studio.run("pip install uv")
|
398
401
|
logger.info(
|
399
402
|
f"Installing requirements: {pipeline_requirements_to_string}"
|
@@ -481,16 +484,25 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
481
484
|
code_path,
|
482
485
|
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
483
486
|
)
|
487
|
+
time.sleep(6)
|
484
488
|
studio.run(
|
485
489
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
486
490
|
)
|
487
491
|
logger.info(
|
488
492
|
"Uploading wheel package and installing dependencies on main studio"
|
489
493
|
)
|
490
|
-
|
491
|
-
|
492
|
-
|
494
|
+
# for some reason uploading file directly to /teamspace/studios/this_studio/.lightning_studio/.studiorc
|
495
|
+
# doesn't work, while other file names can be uploaded just fine.
|
496
|
+
# below is a workaround to copy the file to the correct location.
|
497
|
+
# we first upload it under a different name and then copy it to the
|
498
|
+
# right location.
|
499
|
+
studio.upload_file(env_file_path)
|
500
|
+
time.sleep(6)
|
501
|
+
studio.run(
|
502
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
493
503
|
)
|
504
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
505
|
+
|
494
506
|
studio.run("pip install uv")
|
495
507
|
studio.run(f"uv pip install {requirements}")
|
496
508
|
studio.run("pip install zenml")
|
@@ -557,13 +569,21 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
557
569
|
studio.run(
|
558
570
|
f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
559
571
|
)
|
560
|
-
studio.upload_file(
|
572
|
+
studio.upload_file(
|
573
|
+
code_path,
|
574
|
+
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
575
|
+
)
|
576
|
+
time.sleep(6)
|
561
577
|
studio.run(
|
562
578
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
563
579
|
)
|
564
|
-
studio.upload_file(
|
565
|
-
|
580
|
+
studio.upload_file(env_file_path)
|
581
|
+
time.sleep(6)
|
582
|
+
studio.run(
|
583
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
566
584
|
)
|
585
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
586
|
+
|
567
587
|
studio.run("pip install uv")
|
568
588
|
studio.run(f"uv pip install {details['requirements']}")
|
569
589
|
studio.run("pip install zenml")
|
zenml/model/model.py
CHANGED
@@ -14,7 +14,6 @@
|
|
14
14
|
"""Model user facing interface to pass into pipeline or step."""
|
15
15
|
|
16
16
|
import datetime
|
17
|
-
import time
|
18
17
|
from typing import (
|
19
18
|
TYPE_CHECKING,
|
20
19
|
Any,
|
@@ -28,7 +27,6 @@ from uuid import UUID
|
|
28
27
|
|
29
28
|
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
|
30
29
|
|
31
|
-
from zenml.constants import MAX_RETRIES_FOR_VERSIONED_ENTITY_CREATION
|
32
30
|
from zenml.enums import MetadataResourceTypes, ModelStages
|
33
31
|
from zenml.exceptions import EntityExistsError
|
34
32
|
from zenml.logger import get_logger
|
@@ -527,14 +525,6 @@ class Model(BaseModel):
|
|
527
525
|
data["suppress_class_validation_warnings"] = True
|
528
526
|
return data
|
529
527
|
|
530
|
-
def _validate_config_in_runtime(self) -> "ModelVersionResponse":
|
531
|
-
"""Validate that config doesn't conflict with runtime environment.
|
532
|
-
|
533
|
-
Returns:
|
534
|
-
The model version based on configuration.
|
535
|
-
"""
|
536
|
-
return self._get_or_create_model_version()
|
537
|
-
|
538
528
|
def _get_or_create_model(self) -> "ModelResponse":
|
539
529
|
"""This method should get or create a model from Model Control Plane.
|
540
530
|
|
@@ -678,16 +668,6 @@ class Model(BaseModel):
|
|
678
668
|
if isinstance(self.version, str):
|
679
669
|
self.version = format_name_template(self.version)
|
680
670
|
|
681
|
-
zenml_client = Client()
|
682
|
-
model_version_request = ModelVersionRequest(
|
683
|
-
user=zenml_client.active_user.id,
|
684
|
-
workspace=zenml_client.active_workspace.id,
|
685
|
-
name=str(self.version) if self.version else None,
|
686
|
-
description=self.description,
|
687
|
-
model=model.id,
|
688
|
-
tags=self.tags,
|
689
|
-
)
|
690
|
-
mv_request = ModelVersionRequest.model_validate(model_version_request)
|
691
671
|
try:
|
692
672
|
if self.version or self.model_version_id:
|
693
673
|
model_version = self._get_model_version()
|
@@ -717,60 +697,34 @@ class Model(BaseModel):
|
|
717
697
|
" as an example. You can explore model versions using "
|
718
698
|
f"`zenml model version list -n {self.name}` CLI command."
|
719
699
|
)
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
f"in model `{self.name}`. Retried {retries_made} times. "
|
735
|
-
"This could be driven by exceptionally high concurrency of "
|
736
|
-
"pipeline runs. Please, reach out to us on ZenML Slack for support."
|
737
|
-
) from e
|
738
|
-
# smoothed exponential back-off, it will go as 0.2, 0.3,
|
739
|
-
# 0.45, 0.68, 1.01, 1.52, 2.28, 3.42, 5.13, 7.69, ...
|
740
|
-
sleep = 0.2 * 1.5**i
|
741
|
-
logger.debug(
|
742
|
-
f"Failed to create new model version for "
|
743
|
-
f"model `{self.name}`. Retrying in {sleep}..."
|
744
|
-
)
|
745
|
-
time.sleep(sleep)
|
746
|
-
retries_made += 1
|
747
|
-
self.version = model_version.name
|
700
|
+
|
701
|
+
client = Client()
|
702
|
+
model_version_request = ModelVersionRequest(
|
703
|
+
user=client.active_user.id,
|
704
|
+
workspace=client.active_workspace.id,
|
705
|
+
name=str(self.version) if self.version else None,
|
706
|
+
description=self.description,
|
707
|
+
model=model.id,
|
708
|
+
tags=self.tags,
|
709
|
+
)
|
710
|
+
model_version = client.zen_store.create_model_version(
|
711
|
+
model_version=model_version_request
|
712
|
+
)
|
713
|
+
|
748
714
|
self._created_model_version = True
|
749
715
|
|
750
716
|
logger.info(
|
751
717
|
"Created new model version `%s` for model `%s`.",
|
752
|
-
|
718
|
+
model_version.name,
|
753
719
|
self.name,
|
754
720
|
)
|
755
721
|
|
722
|
+
self.version = model_version.name
|
756
723
|
self.model_version_id = model_version.id
|
757
724
|
self._model_id = model_version.model.id
|
758
725
|
self._number = model_version.number
|
759
726
|
return model_version
|
760
727
|
|
761
|
-
def _merge(self, model: "Model") -> None:
|
762
|
-
self.license = self.license or model.license
|
763
|
-
self.description = self.description or model.description
|
764
|
-
self.audience = self.audience or model.audience
|
765
|
-
self.use_cases = self.use_cases or model.use_cases
|
766
|
-
self.limitations = self.limitations or model.limitations
|
767
|
-
self.trade_offs = self.trade_offs or model.trade_offs
|
768
|
-
self.ethics = self.ethics or model.ethics
|
769
|
-
if model.tags is not None:
|
770
|
-
self.tags = list(
|
771
|
-
{t for t in self.tags or []}.union(set(model.tags))
|
772
|
-
)
|
773
|
-
|
774
728
|
def __hash__(self) -> int:
|
775
729
|
"""Get hash of the `Model`.
|
776
730
|
|
zenml/model/utils.py
CHANGED
@@ -56,6 +56,11 @@ def log_model_metadata(
|
|
56
56
|
ValueError: If no model name/version is provided and the function is not
|
57
57
|
called inside a step with configured `model` in decorator.
|
58
58
|
"""
|
59
|
+
logger.warning(
|
60
|
+
"The `log_model_metadata` function is deprecated and will soon be "
|
61
|
+
"removed. Please use `log_metadata` instead."
|
62
|
+
)
|
63
|
+
|
59
64
|
if model_name and model_version:
|
60
65
|
from zenml import Model
|
61
66
|
|