zenml-nightly 0.68.1.dev20241106__py3-none-any.whl → 0.70.0.dev20241115__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +77 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +2 -0
- zenml/artifacts/external_artifact.py +2 -1
- zenml/artifacts/utils.py +138 -79
- zenml/cli/base.py +4 -4
- zenml/cli/model.py +1 -6
- zenml/cli/stack.py +1 -0
- zenml/client.py +29 -74
- zenml/config/server_config.py +17 -1
- zenml/constants.py +2 -7
- zenml/data_validators/base_data_validator.py +2 -2
- zenml/enums.py +20 -4
- zenml/exceptions.py +4 -0
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +20 -18
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +1 -1
- zenml/integrations/deepchecks/data_validators/deepchecks_data_validator.py +1 -1
- zenml/integrations/evidently/__init__.py +1 -1
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +29 -9
- zenml/integrations/tensorboard/visualizers/tensorboard_visualizer.py +60 -54
- zenml/integrations/vllm/services/vllm_deployment.py +16 -7
- zenml/metadata/lazy_load.py +20 -7
- zenml/model/model.py +17 -64
- zenml/model/utils.py +5 -0
- zenml/models/__init__.py +0 -12
- zenml/models/v2/base/filter.py +121 -8
- zenml/models/v2/core/artifact_version.py +42 -7
- zenml/models/v2/core/model_version.py +26 -5
- zenml/models/v2/core/pipeline_run.py +25 -6
- zenml/models/v2/core/run_metadata.py +2 -217
- zenml/models/v2/core/step_run.py +62 -24
- zenml/orchestrators/base_orchestrator.py +12 -1
- zenml/orchestrators/input_utils.py +44 -19
- zenml/orchestrators/step_launcher.py +4 -3
- zenml/orchestrators/step_run_utils.py +19 -15
- zenml/orchestrators/step_runner.py +21 -13
- zenml/orchestrators/utils.py +45 -26
- zenml/stack_deployments/aws_stack_deployment.py +23 -6
- zenml/stack_deployments/azure_stack_deployment.py +28 -5
- zenml/stack_deployments/gcp_stack_deployment.py +25 -8
- zenml/stack_deployments/stack_deployment.py +3 -5
- zenml/steps/base_step.py +1 -1
- zenml/steps/entrypoint_function_utils.py +3 -5
- zenml/steps/step_context.py +3 -2
- zenml/steps/utils.py +13 -2
- zenml/utils/metadata_utils.py +335 -0
- zenml/zen_server/auth.py +221 -3
- zenml/zen_server/cache.py +208 -0
- zenml/zen_server/dashboard/assets/{404-DT4QRUqN.js → 404-NVXKFp-x.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-DP6vWzyx.js → @radix-DeK6qiuw.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-BMhZulnd.js → @react-router-B3Z5rLr2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-8U9qNlMR.js → @reactflow-CK0KJUen.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-BUCbhJyH.js → @tanstack-DT5WLu9C.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-DezXKmDf.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-CqybNv0k.js → CodeSnippet-JzR8CEtw.js} +2 -2
- zenml/zen_server/dashboard/assets/{CollapsibleCard-0r_8G2Lj.js → CollapsibleCard-DQW_ktMO.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-BDjgBQKi.js → Commands-DL2kwkRd.js} +1 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-D_g62Wv8.js +1 -0
- zenml/zen_server/dashboard/assets/{CopyButton-C745BrKi.js → CopyButton-LNcWaa14.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-PpAq0CeZ.js → CsvVizualization-DknpE5ej.js} +5 -5
- zenml/zen_server/dashboard/assets/{DialogItem-DcVCZEew.js → DialogItem-Bxf8FuAT.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-BeXgUG_C.js → DisplayDate-CDMUcQHS.js} +1 -1
- zenml/zen_server/dashboard/assets/{EmptyState-DeK7H4pr.js → EmptyState-BzdlCwp3.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-BMlzibXj.js → Error-DYflYyps.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-C7zyIQKZ.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-BLf40fLV.js → Helpbox-oYSGpLqd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-BwisKifi.js → Infobox-Cx4xGoXR.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-jEgodSgX.js → InlineAvatar-DiGOWNKF.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-3lLt1ih0.js → Lock-CYYy18Mm.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-8O9kTr-2.js → MarkdownVisualization-ylXaAxev.js} +1 -1
- zenml/zen_server/dashboard/assets/NestedCollapsible-DYbgyKxK.js +1 -0
- zenml/zen_server/dashboard/assets/{NumberBox-T9eELfLZ.js → NumberBox-Dtp3J6g5.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-03iZf8-N.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-CW0kqY0W.js → PasswordChecker-B0nadgh6.js} +1 -1
- zenml/zen_server/dashboard/assets/ProBadge-D_EB8HNo.js +1 -0
- zenml/zen_server/dashboard/assets/ProCta-DqNS4v3x.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderIcon-Bki2aw8w.js +1 -0
- zenml/zen_server/dashboard/assets/{ProviderRadio-BROY1700.js → ProviderRadio-8f43sPD4.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +1 -0
- zenml/zen_server/dashboard/assets/RunsBody-07YEO7qI.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-lp1KgU4e.js +1 -0
- zenml/zen_server/dashboard/assets/{SecretTooltip-C_qByGWB.js → SecretTooltip-CgnbyeOx.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-7pRB00El.js → SetPassword-CpP418A2.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +1 -0
- zenml/zen_server/dashboard/assets/Tabs-BktHkCJJ.js +1 -0
- zenml/zen_server/dashboard/assets/Tick-BlMoIlJT.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DckMEkFf.js → UpdatePasswordSchemas-Sc0A0pP-.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-DVceN14P.js → UsageReason-YYduL4fj.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-CW0Cvd70.js → WizardFooter-dgmizSJC.js} +1 -1
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-D-c2G6lV.js +1 -0
- zenml/zen_server/dashboard/assets/check-DloQpStc.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-Dwxliy1Z.js → check-circle-jNbX5-sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-8wLBS5pQ.js → chevron-down-6JyMkfjR.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-DoD8iXWM.js → chevron-right-double-D7ojK9Co.js} +1 -1
- zenml/zen_server/dashboard/assets/{code-browser-CZUQs3Wa.js → code-browser-CUFUIHfp.js} +1 -1
- zenml/zen_server/dashboard/assets/{copy-CaSMXwiU.js → copy-C8XQA2Ug.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-DM_JPgef.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-BFNgg-z3.js → docker-BdA9vrnW.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-DK5Duzx4.js → dots-horizontal-otGBOSDJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{form-schemas-1AyOCx90.js → form-schemas-K6FYKjwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{gcp-7M2Yf3ZK.js → gcp-CFtm4BA7.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-Dam461dC.js → help-Cc9bBIJH.js} +1 -1
- zenml/zen_server/dashboard/assets/index-B1mVPYxf.js +1 -0
- zenml/zen_server/dashboard/assets/index-BAkC7FXi.js +1 -0
- zenml/zen_server/dashboard/assets/{index-QQb7wQEC.js → index-CCOPpudF.js} +8 -8
- zenml/zen_server/dashboard/assets/index-CEV4Cvaf.js +1 -0
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BVJ8n2-j.js → index-Uu49AX48.js} +1 -1
- zenml/zen_server/dashboard/assets/{index.esm-cuVep_NJ.js → index.esm-Dy6Z9Ung.js} +1 -1
- zenml/zen_server/dashboard/assets/{kubernetes--g7r02Zu.js → kubernetes-B2wmAJ1d.js} +1 -1
- zenml/zen_server/dashboard/assets/{layout-DCSYN7-C.js → layout-BtHBmE4w.js} +1 -1
- zenml/zen_server/dashboard/assets/{link-external-CBEk6kEG.js → link-external-b9AXw_sW.js} +1 -1
- zenml/zen_server/dashboard/assets/{login-mutation-DTcAFP1l.js → login-mutation-hf-lK87O.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D5bdJGur.js → logs-WMSM52RF.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-Cc-JkRH2.js → not-found-BGirLjU-.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-Cs35Szwh.js → package-C6uypY4h.js} +1 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DH_Z7iW1.js → page-6m6yHHlE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BDigxVpo.js +1 -0
- zenml/zen_server/dashboard/assets/page-BR68V0V1.js +1 -0
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BQQKaabe.js → page-BU7huvKw.js} +3 -3
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +1 -0
- zenml/zen_server/dashboard/assets/page-C00YAkaB.js +1 -0
- zenml/zen_server/dashboard/assets/{page-N4qoPHKb.js → page-CD-DcWoy.js} +1 -1
- zenml/zen_server/dashboard/assets/page-COXXJj1k.js +1 -0
- zenml/zen_server/dashboard/assets/page-CbpvrsDL.js +1 -0
- zenml/zen_server/dashboard/assets/page-CdMWnQak.js +1 -0
- zenml/zen_server/dashboard/assets/{page-ClUVkl-O.js → page-CjGdWY13.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLixvR-7.js → page-D01JhjQB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D6uU2ax4.js +1 -0
- zenml/zen_server/dashboard/assets/page-D7S3aCbF.js +1 -0
- zenml/zen_server/dashboard/assets/{page-9yplj5JT.js → page-DLC-bNBP.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DXSTpqRD.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DzpVUZ8f.js → page-DakHVWXF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DIOXwhiD.js → page-Df-Fw0aq.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-B-y2XKIc.js → page-DfbXf_8s.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DjRJCGb3.js +1 -0
- zenml/zen_server/dashboard/assets/{page-C0N5q3l7.js → page-Djikxq_S.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +3 -0
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Vcxara9U.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +14 -0
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +1 -0
- zenml/zen_server/dashboard/assets/page-yYC9OI-E.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-DNb5cdrU.js → persist-Coz7ZWvz.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-CP0JmYZ4.js → persist-GjC8PZoC.js} +1 -1
- zenml/zen_server/dashboard/assets/{plus-C9IxgN2M.js → plus-tf1V2hTJ.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-BVu22P_C.js → refresh-BjOeWlEq.js} +1 -1
- zenml/zen_server/dashboard/assets/{rocket-CONEmRmB.js → rocket-DjT2cDvG.js} +1 -1
- zenml/zen_server/dashboard/assets/sharedSchema-CQb14VSr.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-OPEW-cDJ.js +1 -0
- zenml/zen_server/dashboard/assets/{tick-circle-CM1ZScbQ.js → tick-circle-BEX_Tp4v.js} +1 -1
- zenml/zen_server/dashboard/assets/{trash-DkJHMOg7.js → trash-arLUMWMS.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-DsU8cNVl.js → update-server-settings-mutation-LwuQfHYn.js} +1 -1
- zenml/zen_server/dashboard/assets/upgrade-form-CwRHBuXB.webp +0 -0
- zenml/zen_server/dashboard/assets/url-CkvKAnwF.js +1 -0
- zenml/zen_server/dashboard/assets/{zod-D89GC_vc.js → zod-BwEbpOxH.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/exceptions.py +2 -0
- zenml/zen_server/jwt.py +30 -13
- zenml/zen_server/rbac/endpoint_utils.py +43 -1
- zenml/zen_server/rbac/utils.py +0 -2
- zenml/zen_server/routers/artifact_version_endpoints.py +27 -1
- zenml/zen_server/routers/auth_endpoints.py +134 -102
- zenml/zen_server/routers/logs_endpoints.py +66 -0
- zenml/zen_server/routers/workspaces_endpoints.py +3 -4
- zenml/zen_server/template_execution/utils.py +14 -16
- zenml/zen_server/utils.py +27 -0
- zenml/zen_server/zen_server_api.py +6 -3
- zenml/zen_stores/migrations/versions/0.70.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/1cb6477f72d6_move_artifact_save_type.py +99 -0
- zenml/zen_stores/migrations/versions/904464ea4041_add_pipeline_model_run_unique_constraints.py +192 -0
- zenml/zen_stores/migrations/versions/b557b2871693_update_step_run_input_types.py +33 -0
- zenml/zen_stores/rest_zen_store.py +68 -64
- zenml/zen_stores/schemas/artifact_schemas.py +8 -1
- zenml/zen_stores/schemas/model_schemas.py +27 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +6 -1
- zenml/zen_stores/schemas/pipeline_schemas.py +8 -2
- zenml/zen_stores/schemas/run_metadata_schemas.py +1 -48
- zenml/zen_stores/schemas/step_run_schemas.py +18 -10
- zenml/zen_stores/sql_zen_store.py +283 -219
- zenml/zen_stores/zen_store_interface.py +15 -42
- {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/METADATA +2 -2
- {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/RECORD +194 -179
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-C6N2rGrB.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentBadge-DUiEYJHu.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentFallbackDialog-BFoH5K4V.js +0 -1
- zenml/zen_server/dashboard/assets/ComponentIcon-CAIoUis2.js +0 -1
- zenml/zen_server/dashboard/assets/Partials-YPBB3V4q.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-Bb3Xha5A.js +0 -1
- zenml/zen_server/dashboard/assets/RunSelector-DCiL3M1c.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-DfUiGFVd.js +0 -1
- zenml/zen_server/dashboard/assets/Tick-CykQFPj2.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-B-s_HMDm.js +0 -1
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/create-stack-CEmaPZ4c.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-D-LKbGyz.js +0 -1
- zenml/zen_server/dashboard/assets/index-Bpmj40BI.js +0 -1
- zenml/zen_server/dashboard/assets/index-CbU4Ln_E.css +0 -1
- zenml/zen_server/dashboard/assets/index-DKPhqP2B.js +0 -1
- zenml/zen_server/dashboard/assets/page-BBpOxVcY.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRInM1Lg.js +0 -1
- zenml/zen_server/dashboard/assets/page-BjjlMk7s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bvd7YH2A.js +0 -1
- zenml/zen_server/dashboard/assets/page-CT3Nep8W.js +0 -1
- zenml/zen_server/dashboard/assets/page-C_f47pBf.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cmv8C_yM.js +0 -3
- zenml/zen_server/dashboard/assets/page-CyN2bdWG.js +0 -1
- zenml/zen_server/dashboard/assets/page-CzzXH4fs.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTlGjgnG.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dbpl86h0.js +0 -1
- zenml/zen_server/dashboard/assets/page-Ddgy6kDS.js +0 -1
- zenml/zen_server/dashboard/assets/page-DtCAfBLy.js +0 -9
- zenml/zen_server/dashboard/assets/page-Dx16z7nA.js +0 -1
- zenml/zen_server/dashboard/assets/page-McUyYbo1.js +0 -1
- zenml/zen_server/dashboard/assets/page-T1P3RyAR.js +0 -1
- zenml/zen_server/dashboard/assets/page-bKaULTGG.js +0 -1
- zenml/zen_server/dashboard/assets/page-sbXUJy9t.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-TMLu-nYQ.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-xmYxSsUY.js +0 -1
- zenml/zen_server/dashboard/assets/url-D5le3J4q.js +0 -1
- zenml/zen_server/routers/run_metadata_endpoints.py +0 -96
- {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.68.1.dev20241106.dist-info → zenml_nightly-0.70.0.dev20241115.dist-info}/entry_points.txt +0 -0
zenml/client.py
CHANGED
@@ -136,9 +136,7 @@ from zenml.models import (
|
|
136
136
|
PipelineResponse,
|
137
137
|
PipelineRunFilter,
|
138
138
|
PipelineRunResponse,
|
139
|
-
RunMetadataFilter,
|
140
139
|
RunMetadataRequest,
|
141
|
-
RunMetadataResponse,
|
142
140
|
RunTemplateFilter,
|
143
141
|
RunTemplateRequest,
|
144
142
|
RunTemplateResponse,
|
@@ -190,6 +188,7 @@ from zenml.models import (
|
|
190
188
|
WorkspaceResponse,
|
191
189
|
WorkspaceUpdate,
|
192
190
|
)
|
191
|
+
from zenml.models.v2.core.step_run import StepRunUpdate
|
193
192
|
from zenml.services.service import ServiceConfig
|
194
193
|
from zenml.services.service_status import ServiceState
|
195
194
|
from zenml.services.service_type import ServiceType
|
@@ -3427,7 +3426,9 @@ class Client(metaclass=ClientMetaClass):
|
|
3427
3426
|
Args:
|
3428
3427
|
id_or_prefix: The id or id prefix of the deployment.
|
3429
3428
|
"""
|
3430
|
-
deployment = self.get_deployment(
|
3429
|
+
deployment = self.get_deployment(
|
3430
|
+
id_or_prefix=id_or_prefix, hydrate=False
|
3431
|
+
)
|
3431
3432
|
self.zen_store.delete_deployment(deployment_id=deployment.id)
|
3432
3433
|
|
3433
3434
|
# ------------------------------ Run templates -----------------------------
|
@@ -3795,6 +3796,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3795
3796
|
templatable: Optional[bool] = None,
|
3796
3797
|
tag: Optional[str] = None,
|
3797
3798
|
user: Optional[Union[UUID, str]] = None,
|
3799
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
3798
3800
|
pipeline: Optional[Union[UUID, str]] = None,
|
3799
3801
|
code_repository: Optional[Union[UUID, str]] = None,
|
3800
3802
|
model: Optional[Union[UUID, str]] = None,
|
@@ -3834,6 +3836,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3834
3836
|
templatable: If the runs should be templatable or not.
|
3835
3837
|
tag: Tag to filter by.
|
3836
3838
|
user: The name/ID of the user to filter by.
|
3839
|
+
run_metadata: The run_metadata of the run to filter by.
|
3837
3840
|
pipeline: The name/ID of the pipeline to filter by.
|
3838
3841
|
code_repository: Filter by code repository name/ID.
|
3839
3842
|
model: Filter by model name/ID.
|
@@ -3873,6 +3876,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3873
3876
|
tag=tag,
|
3874
3877
|
unlisted=unlisted,
|
3875
3878
|
user=user,
|
3879
|
+
run_metadata=run_metadata,
|
3876
3880
|
pipeline=pipeline,
|
3877
3881
|
code_repository=code_repository,
|
3878
3882
|
stack=stack,
|
@@ -4166,6 +4170,8 @@ class Client(metaclass=ClientMetaClass):
|
|
4166
4170
|
Returns:
|
4167
4171
|
The artifact version.
|
4168
4172
|
"""
|
4173
|
+
from zenml import get_step_context
|
4174
|
+
|
4169
4175
|
if cll := client_lazy_loader(
|
4170
4176
|
method_name="get_artifact_version",
|
4171
4177
|
name_id_or_prefix=name_id_or_prefix,
|
@@ -4173,13 +4179,26 @@ class Client(metaclass=ClientMetaClass):
|
|
4173
4179
|
hydrate=hydrate,
|
4174
4180
|
):
|
4175
4181
|
return cll # type: ignore[return-value]
|
4176
|
-
|
4182
|
+
|
4183
|
+
artifact = self._get_entity_version_by_id_or_name_or_prefix(
|
4177
4184
|
get_method=self.zen_store.get_artifact_version,
|
4178
4185
|
list_method=self.list_artifact_versions,
|
4179
4186
|
name_id_or_prefix=name_id_or_prefix,
|
4180
4187
|
version=version,
|
4181
4188
|
hydrate=hydrate,
|
4182
4189
|
)
|
4190
|
+
try:
|
4191
|
+
step_run = get_step_context().step_run
|
4192
|
+
client = Client()
|
4193
|
+
client.zen_store.update_run_step(
|
4194
|
+
step_run_id=step_run.id,
|
4195
|
+
step_run_update=StepRunUpdate(
|
4196
|
+
loaded_artifact_versions={artifact.name: artifact.id}
|
4197
|
+
),
|
4198
|
+
)
|
4199
|
+
except RuntimeError:
|
4200
|
+
pass # Cannot link to step run if called outside a step
|
4201
|
+
return artifact
|
4183
4202
|
|
4184
4203
|
def list_artifact_versions(
|
4185
4204
|
self,
|
@@ -4206,6 +4225,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4206
4225
|
user: Optional[Union[UUID, str]] = None,
|
4207
4226
|
model: Optional[Union[UUID, str]] = None,
|
4208
4227
|
pipeline_run: Optional[Union[UUID, str]] = None,
|
4228
|
+
run_metadata: Optional[Dict[str, str]] = None,
|
4209
4229
|
tag: Optional[str] = None,
|
4210
4230
|
hydrate: bool = False,
|
4211
4231
|
) -> Page[ArtifactVersionResponse]:
|
@@ -4237,6 +4257,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4237
4257
|
user: Filter by user name or ID.
|
4238
4258
|
model: Filter by model name or ID.
|
4239
4259
|
pipeline_run: Filter by pipeline run name or ID.
|
4260
|
+
run_metadata: Filter by run metadata.
|
4240
4261
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
4241
4262
|
by including metadata fields in the response.
|
4242
4263
|
|
@@ -4417,7 +4438,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4417
4438
|
resource_id: UUID,
|
4418
4439
|
resource_type: MetadataResourceTypes,
|
4419
4440
|
stack_component_id: Optional[UUID] = None,
|
4420
|
-
) ->
|
4441
|
+
) -> None:
|
4421
4442
|
"""Create run metadata.
|
4422
4443
|
|
4423
4444
|
Args:
|
@@ -4430,7 +4451,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4430
4451
|
the metadata.
|
4431
4452
|
|
4432
4453
|
Returns:
|
4433
|
-
|
4454
|
+
None
|
4434
4455
|
"""
|
4435
4456
|
from zenml.metadata.metadata_types import get_metadata_type
|
4436
4457
|
|
@@ -4465,74 +4486,8 @@ class Client(metaclass=ClientMetaClass):
|
|
4465
4486
|
values=values,
|
4466
4487
|
types=types,
|
4467
4488
|
)
|
4468
|
-
|
4469
|
-
|
4470
|
-
def list_run_metadata(
|
4471
|
-
self,
|
4472
|
-
sort_by: str = "created",
|
4473
|
-
page: int = PAGINATION_STARTING_PAGE,
|
4474
|
-
size: int = PAGE_SIZE_DEFAULT,
|
4475
|
-
logical_operator: LogicalOperators = LogicalOperators.AND,
|
4476
|
-
id: Optional[Union[UUID, str]] = None,
|
4477
|
-
created: Optional[Union[datetime, str]] = None,
|
4478
|
-
updated: Optional[Union[datetime, str]] = None,
|
4479
|
-
workspace_id: Optional[UUID] = None,
|
4480
|
-
user_id: Optional[UUID] = None,
|
4481
|
-
resource_id: Optional[UUID] = None,
|
4482
|
-
resource_type: Optional[MetadataResourceTypes] = None,
|
4483
|
-
stack_component_id: Optional[UUID] = None,
|
4484
|
-
key: Optional[str] = None,
|
4485
|
-
value: Optional["MetadataType"] = None,
|
4486
|
-
type: Optional[str] = None,
|
4487
|
-
hydrate: bool = False,
|
4488
|
-
) -> Page[RunMetadataResponse]:
|
4489
|
-
"""List run metadata.
|
4490
|
-
|
4491
|
-
Args:
|
4492
|
-
sort_by: The field to sort the results by.
|
4493
|
-
page: The page number to return.
|
4494
|
-
size: The number of results to return per page.
|
4495
|
-
logical_operator: The logical operator to use for filtering.
|
4496
|
-
id: The ID of the metadata.
|
4497
|
-
created: The creation time of the metadata.
|
4498
|
-
updated: The last update time of the metadata.
|
4499
|
-
workspace_id: The ID of the workspace the metadata belongs to.
|
4500
|
-
user_id: The ID of the user that created the metadata.
|
4501
|
-
resource_id: The ID of the resource the metadata belongs to.
|
4502
|
-
resource_type: The type of the resource the metadata belongs to.
|
4503
|
-
stack_component_id: The ID of the stack component that produced
|
4504
|
-
the metadata.
|
4505
|
-
key: The key of the metadata.
|
4506
|
-
value: The value of the metadata.
|
4507
|
-
type: The type of the metadata.
|
4508
|
-
hydrate: Flag deciding whether to hydrate the output model(s)
|
4509
|
-
by including metadata fields in the response.
|
4510
|
-
|
4511
|
-
Returns:
|
4512
|
-
The run metadata.
|
4513
|
-
"""
|
4514
|
-
metadata_filter_model = RunMetadataFilter(
|
4515
|
-
sort_by=sort_by,
|
4516
|
-
page=page,
|
4517
|
-
size=size,
|
4518
|
-
logical_operator=logical_operator,
|
4519
|
-
id=id,
|
4520
|
-
created=created,
|
4521
|
-
updated=updated,
|
4522
|
-
workspace_id=workspace_id,
|
4523
|
-
user_id=user_id,
|
4524
|
-
resource_id=resource_id,
|
4525
|
-
resource_type=resource_type,
|
4526
|
-
stack_component_id=stack_component_id,
|
4527
|
-
key=key,
|
4528
|
-
value=value,
|
4529
|
-
type=type,
|
4530
|
-
)
|
4531
|
-
metadata_filter_model.set_scope_workspace(self.active_workspace.id)
|
4532
|
-
return self.zen_store.list_run_metadata(
|
4533
|
-
metadata_filter_model,
|
4534
|
-
hydrate=hydrate,
|
4535
|
-
)
|
4489
|
+
self.zen_store.create_run_metadata(run_metadata)
|
4490
|
+
return None
|
4536
4491
|
|
4537
4492
|
# -------------------------------- Secrets ---------------------------------
|
4538
4493
|
|
zenml/config/server_config.py
CHANGED
@@ -19,13 +19,14 @@ from secrets import token_hex
|
|
19
19
|
from typing import Any, Dict, List, Optional, Union
|
20
20
|
from uuid import UUID
|
21
21
|
|
22
|
-
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
22
|
+
from pydantic import BaseModel, ConfigDict, Field, PositiveInt, model_validator
|
23
23
|
|
24
24
|
from zenml.constants import (
|
25
25
|
DEFAULT_ZENML_JWT_TOKEN_ALGORITHM,
|
26
26
|
DEFAULT_ZENML_JWT_TOKEN_LEEWAY,
|
27
27
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING,
|
28
28
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT,
|
29
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME,
|
29
30
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY,
|
30
31
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE,
|
31
32
|
DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS,
|
@@ -119,6 +120,8 @@ class ServerConfiguration(BaseModel):
|
|
119
120
|
time of the JWT tokens issued to clients after they have
|
120
121
|
authenticated with the ZenML server using an OAuth 2.0 device
|
121
122
|
that has been marked as trusted.
|
123
|
+
generic_api_token_lifetime: The lifetime in seconds that generic
|
124
|
+
short-lived API tokens issued for automation purposes are valid.
|
122
125
|
external_login_url: The login URL of an external authenticator service
|
123
126
|
to use with the `EXTERNAL` authentication scheme.
|
124
127
|
external_user_info_url: The user info URL of an external authenticator
|
@@ -230,6 +233,12 @@ class ServerConfiguration(BaseModel):
|
|
230
233
|
deployment.
|
231
234
|
max_request_body_size_in_bytes: The maximum size of the request body in
|
232
235
|
bytes. If not specified, the default value of 256 Kb will be used.
|
236
|
+
memcache_max_capacity: The maximum number of entries that the memory
|
237
|
+
cache can hold. If not specified, the default value of 1000 will be
|
238
|
+
used.
|
239
|
+
memcache_default_expiry: The default expiry time in seconds for cache
|
240
|
+
entries. If not specified, the default value of 30 seconds will be
|
241
|
+
used.
|
233
242
|
"""
|
234
243
|
|
235
244
|
deployment_type: ServerDeploymentType = ServerDeploymentType.OTHER
|
@@ -257,6 +266,10 @@ class ServerConfiguration(BaseModel):
|
|
257
266
|
device_expiration_minutes: Optional[int] = None
|
258
267
|
trusted_device_expiration_minutes: Optional[int] = None
|
259
268
|
|
269
|
+
generic_api_token_lifetime: PositiveInt = (
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME
|
271
|
+
)
|
272
|
+
|
260
273
|
external_login_url: Optional[str] = None
|
261
274
|
external_user_info_url: Optional[str] = None
|
262
275
|
external_cookie_name: Optional[str] = None
|
@@ -321,6 +334,9 @@ class ServerConfiguration(BaseModel):
|
|
321
334
|
DEFAULT_ZENML_SERVER_MAX_REQUEST_BODY_SIZE_IN_BYTES
|
322
335
|
)
|
323
336
|
|
337
|
+
memcache_max_capacity: int = 1000
|
338
|
+
memcache_default_expiry: int = 30
|
339
|
+
|
324
340
|
_deployment_id: Optional[UUID] = None
|
325
341
|
|
326
342
|
@model_validator(mode="before")
|
zenml/constants.py
CHANGED
@@ -167,9 +167,6 @@ ENV_ZENML_SERVER = "ZENML_SERVER"
|
|
167
167
|
ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
|
168
168
|
ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS"
|
169
169
|
ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
|
170
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES = (
|
171
|
-
"ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES"
|
172
|
-
)
|
173
170
|
ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
|
174
171
|
ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
|
175
172
|
ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
|
@@ -270,6 +267,7 @@ ZENML_API_KEY_PREFIX = "ZENKEY_"
|
|
270
267
|
DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48 # 48 hours
|
271
268
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
|
272
269
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY = 1000
|
270
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME = 60 * 60 # 1 hour
|
273
271
|
|
274
272
|
DEFAULT_ZENML_SERVER_SECURE_HEADERS_HSTS = (
|
275
273
|
"max-age=63072000; includeSubdomains"
|
@@ -338,6 +336,7 @@ ARTIFACTS = "/artifacts"
|
|
338
336
|
ARTIFACT_VERSIONS = "/artifact_versions"
|
339
337
|
ARTIFACT_VISUALIZATIONS = "/artifact_visualizations"
|
340
338
|
AUTH = "/auth"
|
339
|
+
BATCH = "/batch"
|
341
340
|
CODE_REFERENCES = "/code_references"
|
342
341
|
CODE_REPOSITORIES = "/code_repositories"
|
343
342
|
COMPONENT_TYPES = "/component-types"
|
@@ -409,10 +408,6 @@ MODEL_METADATA_YAML_FILE_NAME = "model_metadata.yaml"
|
|
409
408
|
|
410
409
|
# orchestrator constants
|
411
410
|
ORCHESTRATOR_DOCKER_IMAGE_KEY = "orchestrator"
|
412
|
-
PIPELINE_API_TOKEN_EXPIRES_MINUTES = handle_int_env_var(
|
413
|
-
ENV_ZENML_PIPELINE_API_TOKEN_EXPIRES_MINUTES,
|
414
|
-
default=60 * 24, # 24 hours
|
415
|
-
)
|
416
411
|
|
417
412
|
# Secret constants
|
418
413
|
SECRET_VALUES = "values"
|
@@ -175,7 +175,7 @@ class BaseDataValidator(StackComponent):
|
|
175
175
|
|
176
176
|
This method should be implemented by data validators that support
|
177
177
|
running model validation checks (e.g. confusion matrix validation,
|
178
|
-
performance reports, model error
|
178
|
+
performance reports, model error analyzes, etc).
|
179
179
|
|
180
180
|
Unlike `data_validation`, model validation checks require that a model
|
181
181
|
be present as an active component during the validation process.
|
@@ -184,7 +184,7 @@ class BaseDataValidator(StackComponent):
|
|
184
184
|
accommodate different categories of data validation tests, e.g.:
|
185
185
|
|
186
186
|
* single dataset tests: confusion matrix validation,
|
187
|
-
performance reports, model error
|
187
|
+
performance reports, model error analyzes, etc
|
188
188
|
* model comparison tests: tests that identify changes in a model
|
189
189
|
behavior by comparing how it performs on two different datasets.
|
190
190
|
|
zenml/enums.py
CHANGED
@@ -34,15 +34,23 @@ class ArtifactType(StrEnum):
|
|
34
34
|
class StepRunInputArtifactType(StrEnum):
|
35
35
|
"""All possible types of a step run input artifact."""
|
36
36
|
|
37
|
-
|
37
|
+
STEP_OUTPUT = (
|
38
|
+
"step_output" # input argument that is the output of a previous step
|
39
|
+
)
|
38
40
|
MANUAL = "manual" # manually loaded via `zenml.load_artifact()`
|
41
|
+
EXTERNAL = "external" # loaded via `ExternalArtifact(value=...)`
|
42
|
+
LAZY_LOADED = "lazy" # loaded via various lazy methods
|
39
43
|
|
40
44
|
|
41
|
-
class
|
42
|
-
"""All possible types of
|
45
|
+
class ArtifactSaveType(StrEnum):
|
46
|
+
"""All possible method types of how artifact versions can be saved."""
|
43
47
|
|
44
|
-
|
48
|
+
STEP_OUTPUT = "step_output" # output of the current step
|
45
49
|
MANUAL = "manual" # manually saved via `zenml.save_artifact()`
|
50
|
+
PREEXISTING = "preexisting" # register via `zenml.register_artifact()`
|
51
|
+
EXTERNAL = (
|
52
|
+
"external" # saved via `zenml.ExternalArtifact.upload_by_value()`
|
53
|
+
)
|
46
54
|
|
47
55
|
|
48
56
|
class VisualizationType(StrEnum):
|
@@ -237,6 +245,13 @@ class OAuthDeviceStatus(StrEnum):
|
|
237
245
|
LOCKED = "locked"
|
238
246
|
|
239
247
|
|
248
|
+
class APITokenType(StrEnum):
|
249
|
+
"""The API token type."""
|
250
|
+
|
251
|
+
GENERIC = "generic"
|
252
|
+
WORKLOAD = "workload"
|
253
|
+
|
254
|
+
|
240
255
|
class GenericFilterOps(StrEnum):
|
241
256
|
"""Ops for all filters for string values on list methods."""
|
242
257
|
|
@@ -245,6 +260,7 @@ class GenericFilterOps(StrEnum):
|
|
245
260
|
CONTAINS = "contains"
|
246
261
|
STARTSWITH = "startswith"
|
247
262
|
ENDSWITH = "endswith"
|
263
|
+
ONEOF = "oneof"
|
248
264
|
GTE = "gte"
|
249
265
|
GT = "gt"
|
250
266
|
LTE = "lte"
|
zenml/exceptions.py
CHANGED
@@ -168,6 +168,10 @@ class EntityExistsError(ZenMLBaseException):
|
|
168
168
|
"""Raised when trying to register an entity that already exists."""
|
169
169
|
|
170
170
|
|
171
|
+
class EntityCreationError(ZenMLBaseException, RuntimeError):
|
172
|
+
"""Raised when failing to create an entity."""
|
173
|
+
|
174
|
+
|
171
175
|
class ActionExistsError(EntityExistsError):
|
172
176
|
"""Raised when registering an action with a name that already exists."""
|
173
177
|
|
@@ -478,7 +478,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
478
478
|
)
|
479
479
|
|
480
480
|
# Yield metadata based on the generated execution object
|
481
|
-
yield from self.compute_metadata(
|
481
|
+
yield from self.compute_metadata(
|
482
|
+
execution=execution, settings=settings
|
483
|
+
)
|
482
484
|
|
483
485
|
# mainly for testing purposes, we wait for the pipeline to finish
|
484
486
|
if settings.synchronous:
|
@@ -518,19 +520,6 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
518
520
|
"pipeline_execution_arn": pipeline_execution_arn,
|
519
521
|
}
|
520
522
|
|
521
|
-
aws_run_id = os.environ[ENV_ZENML_SAGEMAKER_RUN_ID].split("/")[-1]
|
522
|
-
|
523
|
-
region_name, _, _ = dissect_pipeline_execution_arn(
|
524
|
-
pipeline_execution_arn=pipeline_execution_arn
|
525
|
-
)
|
526
|
-
|
527
|
-
orchestrator_logs_url = (
|
528
|
-
f"https://{region_name}.console.aws.amazon.com/"
|
529
|
-
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
530
|
-
f"/$252Faws$252Fsagemaker$252FProcessingJobs$3FlogStreamNameFilter"
|
531
|
-
f"$3Dpipelines-{aws_run_id}-"
|
532
|
-
)
|
533
|
-
run_metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_logs_url)
|
534
523
|
return run_metadata
|
535
524
|
|
536
525
|
def fetch_status(self, run: "PipelineRunResponse") -> ExecutionStatus:
|
@@ -566,7 +555,7 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
566
555
|
|
567
556
|
# Fetch the status of the _PipelineExecution
|
568
557
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
569
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
558
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
570
559
|
elif run.orchestrator_run_id is not None:
|
571
560
|
run_id = run.orchestrator_run_id
|
572
561
|
else:
|
@@ -590,12 +579,15 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
590
579
|
raise ValueError("Unknown status for the pipeline execution.")
|
591
580
|
|
592
581
|
def compute_metadata(
|
593
|
-
self,
|
582
|
+
self,
|
583
|
+
execution: Any,
|
584
|
+
settings: SagemakerOrchestratorSettings,
|
594
585
|
) -> Iterator[Dict[str, MetadataType]]:
|
595
586
|
"""Generate run metadata based on the generated Sagemaker Execution.
|
596
587
|
|
597
588
|
Args:
|
598
589
|
execution: The corresponding _PipelineExecution object.
|
590
|
+
settings: The Sagemaker orchestrator settings.
|
599
591
|
|
600
592
|
Yields:
|
601
593
|
A dictionary of metadata related to the pipeline run.
|
@@ -612,7 +604,9 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
612
604
|
metadata[METADATA_ORCHESTRATOR_URL] = Uri(orchestrator_url)
|
613
605
|
|
614
606
|
# URL to the corresponding CloudWatch page
|
615
|
-
if logs_url := self._compute_orchestrator_logs_url(
|
607
|
+
if logs_url := self._compute_orchestrator_logs_url(
|
608
|
+
execution, settings
|
609
|
+
):
|
616
610
|
metadata[METADATA_ORCHESTRATOR_LOGS_URL] = Uri(logs_url)
|
617
611
|
|
618
612
|
yield metadata
|
@@ -656,11 +650,13 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
656
650
|
@staticmethod
|
657
651
|
def _compute_orchestrator_logs_url(
|
658
652
|
pipeline_execution: Any,
|
653
|
+
settings: SagemakerOrchestratorSettings,
|
659
654
|
) -> Optional[str]:
|
660
655
|
"""Generate the CloudWatch URL upon pipeline execution.
|
661
656
|
|
662
657
|
Args:
|
663
658
|
pipeline_execution: The corresponding _PipelineExecution object.
|
659
|
+
settings: The Sagemaker orchestrator settings.
|
664
660
|
|
665
661
|
Returns:
|
666
662
|
the URL querying the pipeline logs in CloudWatch on AWS.
|
@@ -670,10 +666,16 @@ class SagemakerOrchestrator(ContainerizedOrchestrator):
|
|
670
666
|
pipeline_execution.arn
|
671
667
|
)
|
672
668
|
|
669
|
+
use_training_jobs = True
|
670
|
+
if settings.use_training_step is not None:
|
671
|
+
use_training_jobs = settings.use_training_step
|
672
|
+
|
673
|
+
job_type = "Training" if use_training_jobs else "Processing"
|
674
|
+
|
673
675
|
return (
|
674
676
|
f"https://{region_name}.console.aws.amazon.com/"
|
675
677
|
f"cloudwatch/home?region={region_name}#logsV2:log-groups/log-group"
|
676
|
-
f"/$252Faws$252Fsagemaker$
|
678
|
+
f"/$252Faws$252Fsagemaker$252F{job_type}Jobs$3FlogStreamNameFilter"
|
677
679
|
f"$3Dpipelines-{execution_id}-"
|
678
680
|
)
|
679
681
|
except Exception as e:
|
@@ -482,7 +482,7 @@ class AzureMLOrchestrator(ContainerizedOrchestrator):
|
|
482
482
|
|
483
483
|
# Fetch the status of the PipelineJob
|
484
484
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
485
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
485
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
486
486
|
elif run.orchestrator_run_id is not None:
|
487
487
|
run_id = run.orchestrator_run_id
|
488
488
|
else:
|
@@ -430,7 +430,7 @@ class DeepchecksDataValidator(BaseDataValidator):
|
|
430
430
|
"""Run one or more Deepchecks model validation checks.
|
431
431
|
|
432
432
|
Call this method to perform model validation checks (e.g. confusion
|
433
|
-
matrix validation, performance reports, model error
|
433
|
+
matrix validation, performance reports, model error analyzes, etc).
|
434
434
|
A second dataset is required for model performance comparison tests
|
435
435
|
(i.e. tests that identify changes in a model behavior by comparing how
|
436
436
|
it performs on two different datasets).
|
@@ -33,7 +33,7 @@ from zenml.stack import Flavor
|
|
33
33
|
|
34
34
|
# Fix numba errors in Docker and suppress logs and deprecation warning spam
|
35
35
|
try:
|
36
|
-
from numba.core.errors import (
|
36
|
+
from numba.core.errors import (
|
37
37
|
NumbaDeprecationWarning,
|
38
38
|
NumbaPendingDeprecationWarning,
|
39
39
|
)
|
@@ -835,7 +835,7 @@ class VertexOrchestrator(ContainerizedOrchestrator, GoogleCredentialsMixin):
|
|
835
835
|
|
836
836
|
# Fetch the status of the PipelineJob
|
837
837
|
if METADATA_ORCHESTRATOR_RUN_ID in run.run_metadata:
|
838
|
-
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
838
|
+
run_id = run.run_metadata[METADATA_ORCHESTRATOR_RUN_ID]
|
839
839
|
elif run.orchestrator_run_id is not None:
|
840
840
|
run_id = run.orchestrator_run_id
|
841
841
|
else:
|
@@ -390,10 +390,13 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
390
390
|
studio.run(
|
391
391
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
392
392
|
)
|
393
|
-
studio.upload_file(
|
394
|
-
|
395
|
-
|
393
|
+
studio.upload_file(env_file_path)
|
394
|
+
time.sleep(6)
|
395
|
+
studio.run(
|
396
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
396
397
|
)
|
398
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
399
|
+
|
397
400
|
studio.run("pip install uv")
|
398
401
|
logger.info(
|
399
402
|
f"Installing requirements: {pipeline_requirements_to_string}"
|
@@ -481,16 +484,25 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
481
484
|
code_path,
|
482
485
|
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
483
486
|
)
|
487
|
+
time.sleep(6)
|
484
488
|
studio.run(
|
485
489
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
486
490
|
)
|
487
491
|
logger.info(
|
488
492
|
"Uploading wheel package and installing dependencies on main studio"
|
489
493
|
)
|
490
|
-
|
491
|
-
|
492
|
-
|
494
|
+
# for some reason uploading file directly to /teamspace/studios/this_studio/.lightning_studio/.studiorc
|
495
|
+
# doesn't work, while other file names can be uploaded just fine.
|
496
|
+
# below is a workaround to copy the file to the correct location.
|
497
|
+
# we first upload it under a different name and then copy it to the
|
498
|
+
# right location.
|
499
|
+
studio.upload_file(env_file_path)
|
500
|
+
time.sleep(6)
|
501
|
+
studio.run(
|
502
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
493
503
|
)
|
504
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
505
|
+
|
494
506
|
studio.run("pip install uv")
|
495
507
|
studio.run(f"uv pip install {requirements}")
|
496
508
|
studio.run("pip install zenml")
|
@@ -557,13 +569,21 @@ class LightningOrchestrator(WheeledOrchestrator):
|
|
557
569
|
studio.run(
|
558
570
|
f"mkdir -p /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
559
571
|
)
|
560
|
-
studio.upload_file(
|
572
|
+
studio.upload_file(
|
573
|
+
code_path,
|
574
|
+
remote_path=f"/teamspace/studios/this_studio/zenml_codes/{filename}",
|
575
|
+
)
|
576
|
+
time.sleep(6)
|
561
577
|
studio.run(
|
562
578
|
f"tar -xvzf /teamspace/studios/this_studio/zenml_codes/{filename} -C /teamspace/studios/this_studio/zenml_codes/{filename.rsplit('.', 2)[0]}"
|
563
579
|
)
|
564
|
-
studio.upload_file(
|
565
|
-
|
580
|
+
studio.upload_file(env_file_path)
|
581
|
+
time.sleep(6)
|
582
|
+
studio.run(
|
583
|
+
f"cp {env_file_path.split('/')[-1]} ./.lightning_studio/.studiorc"
|
566
584
|
)
|
585
|
+
studio.run(f"rm {env_file_path.split('/')[-1]}")
|
586
|
+
|
567
587
|
studio.run("pip install uv")
|
568
588
|
studio.run(f"uv pip install {details['requirements']}")
|
569
589
|
studio.run("pip install zenml")
|