zenml-nightly 0.70.0.dev20241201__py3-none-any.whl → 0.71.0.dev20241220__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +4 -4
- RELEASE_NOTES.md +112 -0
- zenml/VERSION +1 -1
- zenml/artifacts/artifact_config.py +8 -5
- zenml/artifacts/utils.py +3 -1
- zenml/cli/__init__.py +4 -4
- zenml/cli/base.py +1 -1
- zenml/cli/pipeline.py +48 -79
- zenml/cli/server.py +19 -19
- zenml/client.py +54 -2
- zenml/config/secret_reference_mixin.py +1 -1
- zenml/config/server_config.py +4 -0
- zenml/constants.py +10 -0
- zenml/image_builders/base_image_builder.py +5 -2
- zenml/image_builders/build_context.py +7 -16
- zenml/integrations/aws/__init__.py +3 -0
- zenml/integrations/aws/flavors/__init__.py +6 -0
- zenml/integrations/aws/flavors/aws_image_builder_flavor.py +146 -0
- zenml/integrations/aws/image_builders/__init__.py +20 -0
- zenml/integrations/aws/image_builders/aws_image_builder.py +307 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +1 -1
- zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +2 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +11 -0
- zenml/integrations/kubernetes/step_operators/kubernetes_step_operator.py +0 -1
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +11 -0
- zenml/integrations/neptune/experiment_trackers/neptune_experiment_tracker.py +7 -5
- zenml/integrations/neptune/experiment_trackers/run_state.py +69 -53
- zenml/integrations/registry.py +2 -2
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +12 -0
- zenml/integrations/wandb/flavors/wandb_experiment_tracker_flavor.py +13 -5
- zenml/materializers/built_in_materializer.py +1 -1
- zenml/model/model.py +12 -16
- zenml/model/utils.py +3 -1
- zenml/models/v2/base/filter.py +26 -30
- zenml/models/v2/base/scoped.py +258 -5
- zenml/models/v2/core/artifact_version.py +15 -26
- zenml/models/v2/core/code_repository.py +1 -12
- zenml/models/v2/core/component.py +5 -46
- zenml/models/v2/core/flavor.py +1 -11
- zenml/models/v2/core/model.py +1 -57
- zenml/models/v2/core/model_version.py +5 -33
- zenml/models/v2/core/model_version_artifact.py +11 -3
- zenml/models/v2/core/model_version_pipeline_run.py +14 -3
- zenml/models/v2/core/pipeline.py +47 -55
- zenml/models/v2/core/pipeline_build.py +67 -12
- zenml/models/v2/core/pipeline_deployment.py +0 -10
- zenml/models/v2/core/pipeline_run.py +91 -29
- zenml/models/v2/core/run_template.py +21 -29
- zenml/models/v2/core/schedule.py +0 -10
- zenml/models/v2/core/secret.py +0 -14
- zenml/models/v2/core/service.py +9 -16
- zenml/models/v2/core/service_connector.py +0 -11
- zenml/models/v2/core/stack.py +21 -30
- zenml/models/v2/core/step_run.py +18 -14
- zenml/models/v2/core/trigger.py +19 -3
- zenml/orchestrators/base_orchestrator.py +13 -1
- zenml/orchestrators/output_utils.py +5 -1
- zenml/orchestrators/step_launcher.py +9 -13
- zenml/orchestrators/step_run_utils.py +8 -204
- zenml/orchestrators/utils.py +55 -27
- zenml/pipelines/build_utils.py +12 -0
- zenml/service_connectors/service_connector_utils.py +3 -9
- zenml/stack/stack_component.py +1 -1
- zenml/stack_deployments/aws_stack_deployment.py +22 -0
- zenml/utils/archivable.py +65 -36
- zenml/utils/code_utils.py +8 -4
- zenml/utils/docker_utils.py +9 -0
- zenml/zen_server/auth.py +9 -10
- zenml/zen_server/dashboard/assets/{404-NVXKFp-x.js → 404-Cqu3EDCm.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CK0KJUen.js → @reactflow-D2Y7BWwz.js} +1 -1
- zenml/zen_server/dashboard/assets/{AlertDialogDropdownItem-DezXKmDf.js → AlertDialogDropdownItem-BHd71pVS.js} +1 -1
- zenml/zen_server/dashboard/assets/{CodeSnippet-JzR8CEtw.js → CodeSnippet-DIonwetW.js} +1 -1
- zenml/zen_server/dashboard/assets/{CollapsibleCard-DQW_ktMO.js → CollapsibleCard-CDnC97pB.js} +1 -1
- zenml/zen_server/dashboard/assets/{Commands-DL2kwkRd.js → Commands-BVEXKAOj.js} +1 -1
- zenml/zen_server/dashboard/assets/{ComponentBadge-D_g62Wv8.js → ComponentBadge-CrRvovox.js} +1 -1
- zenml/zen_server/dashboard/assets/{CopyButton-LNcWaa14.js → CopyButton-B6wGAhQv.js} +1 -1
- zenml/zen_server/dashboard/assets/{CsvVizualization-DknpE5ej.js → CsvVizualization-CjcT7LMm.js} +5 -5
- zenml/zen_server/dashboard/assets/DeleteAlertDialog-D2ELtM2W.js +1 -0
- zenml/zen_server/dashboard/assets/{DialogItem-Bxf8FuAT.js → DialogItem-DXIMhBgU.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-DYflYyps.js → Error-B8uUfTpL.js} +1 -1
- zenml/zen_server/dashboard/assets/{ExecutionStatus-C7zyIQKZ.js → ExecutionStatus-ibAdY-dG.js} +1 -1
- zenml/zen_server/dashboard/assets/{Helpbox-oYSGpLqd.js → Helpbox-BfAfhKHw.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-Cx4xGoXR.js → Infobox-M_SMOu96.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-DiGOWNKF.js → InlineAvatar-DBA0a0-a.js} +1 -1
- zenml/zen_server/dashboard/assets/{NestedCollapsible-DYbgyKxK.js → NestedCollapsible-DpgmEFKw.js} +1 -1
- zenml/zen_server/dashboard/assets/{Partials-03iZf8-N.js → Partials-D_ldD9if.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProBadge-D_EB8HNo.js → ProBadge-DQbfFotM.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProCta-DqNS4v3x.js → ProCta-Bcpb4rcY.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProviderIcon-Bki2aw8w.js → ProviderIcon-BZpgPigN.js} +1 -1
- zenml/zen_server/dashboard/assets/{ProviderRadio-8f43sPD4.js → ProviderRadio-DWPnMuQ1.js} +1 -1
- zenml/zen_server/dashboard/assets/RunSelector-DgRGaAc6.js +1 -0
- zenml/zen_server/dashboard/assets/{RunsBody-07YEO7qI.js → RunsBody-KecfSkjY.js} +1 -1
- zenml/zen_server/dashboard/assets/{SearchField-lp1KgU4e.js → SearchField-n-ILHnaP.js} +1 -1
- zenml/zen_server/dashboard/assets/{SecretTooltip-CgnbyeOx.js → SecretTooltip-B8MrX5yu.js} +1 -1
- zenml/zen_server/dashboard/assets/{SetPassword-CpP418A2.js → SetPassword-B_IVq_wg.js} +1 -1
- zenml/zen_server/dashboard/assets/StackList-TWPBYnkF.js +1 -0
- zenml/zen_server/dashboard/assets/{Tabs-BktHkCJJ.js → Tabs-Rg857zmd.js} +1 -1
- zenml/zen_server/dashboard/assets/{Tick-BlMoIlJT.js → Tick-COg4A-xo.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-Sc0A0pP-.js → UpdatePasswordSchemas-C6Aj3hm6.js} +1 -1
- zenml/zen_server/dashboard/assets/{UsageReason-YYduL4fj.js → UsageReason-BTLbx7w4.js} +1 -1
- zenml/zen_server/dashboard/assets/{WizardFooter-dgmizSJC.js → WizardFooter-BCAj69Vj.js} +1 -1
- zenml/zen_server/dashboard/assets/{all-pipeline-runs-query-D-c2G6lV.js → all-pipeline-runs-query-DMXkDrV2.js} +1 -1
- zenml/zen_server/dashboard/assets/code-snippets-CqONne41.js +13 -0
- zenml/zen_server/dashboard/assets/{create-stack-DM_JPgef.js → create-stack-HfdbhLs4.js} +1 -1
- zenml/zen_server/dashboard/assets/dates-3pMLCNrD.js +1 -0
- zenml/zen_server/dashboard/assets/delete-run-DZ4hIXff.js +1 -0
- zenml/zen_server/dashboard/assets/{form-schemas-K6FYKjwa.js → form-schemas-B0AVEd9b.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-BAkC7FXi.js → index-DPqSWjug.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-CEV4Cvaf.js → index-DScjfBRb.js} +1 -1
- zenml/zen_server/dashboard/assets/index-DXvT1_Um.css +1 -0
- zenml/zen_server/dashboard/assets/{index-CCOPpudF.js → index-FO-p0GU7.js} +5 -5
- zenml/zen_server/dashboard/assets/{index-B1mVPYxf.js → index-I3bKUGUj.js} +1 -1
- zenml/zen_server/dashboard/assets/key-icon-aH-QIa5R.js +1 -0
- zenml/zen_server/dashboard/assets/login-command-CkqxPtV3.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-hf-lK87O.js → login-mutation-BQeo4wTY.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-BGirLjU-.js → not-found-gAJ5aDdR.js} +1 -1
- zenml/zen_server/dashboard/assets/page-9Y9-gig0.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DjRJCGb3.js → page-AUwiQ14W.js} +1 -1
- zenml/zen_server/dashboard/assets/page-B6XU7yYT.js +2 -0
- zenml/zen_server/dashboard/assets/{page-C00YAkaB.js → page-BKZYc2Zv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-CdMWnQak.js → page-BU9FG4sR.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-D7S3aCbF.js → page-B_Apk3xg.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-Djikxq_S.js → page-BdowiCbr.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Bg8OjTRe.js +1 -0
- zenml/zen_server/dashboard/assets/page-BxL4qD4_.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DakHVWXF.js → page-CWxT5K5J.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CXuQufSe.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLC-bNBP.js → page-CcQr8CPP.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-CD-DcWoy.js → page-Ce4Hrjnr.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CiYxgZP_.js +1 -0
- zenml/zen_server/dashboard/assets/page-Cldq1mpe.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BDigxVpo.js → page-D4wdonLm.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-D6uU2ax4.js → page-D8ObrbH8.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DXSTpqRD.js → page-DFuAUGt4.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-CbpvrsDL.js → page-DGazBpuP.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-COXXJj1k.js → page-DO1UcqPX.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DRYXdL5o.js +1 -0
- zenml/zen_server/dashboard/assets/{page-Df-Fw0aq.js → page-DYEquBC2.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Dk32IeZm.js +1 -0
- zenml/zen_server/dashboard/assets/{page-yYC9OI-E.js → page-I3nKFGie.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-6m6yHHlE.js → page-M0w-n6vn.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-Vcxara9U.js → page-R5dx3xGF.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BR68V0V1.js → page-bT5pOvcB.js} +1 -1
- zenml/zen_server/dashboard/assets/page-hUqK889I.js +6 -0
- zenml/zen_server/dashboard/assets/{page-CjGdWY13.js → page-h_Stveon.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-D01JhjQB.js → page-r8XK5vR7.js} +1 -1
- zenml/zen_server/dashboard/assets/page-u_-ZXBKb.js +1 -0
- zenml/zen_server/dashboard/assets/page-zaMqB_ao.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-GjC8PZoC.js → persist-AppN1B0J.js} +1 -1
- zenml/zen_server/dashboard/assets/{persist-Coz7ZWvz.js → persist-DAUi_3za.js} +1 -1
- zenml/zen_server/dashboard/assets/service-BqqeXLEe.js +2 -0
- zenml/zen_server/dashboard/assets/{sharedSchema-CQb14VSr.js → sharedSchema-uXN9FLLk.js} +1 -1
- zenml/zen_server/dashboard/assets/{stack-detail-query-OPEW-cDJ.js → stack-detail-query-XfZBiBP2.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-LwuQfHYn.js → update-server-settings-mutation-BWmgVJwA.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-CkvKAnwF.js → url-BLwMbzES.js} +1 -1
- zenml/zen_server/dashboard/index.html +4 -4
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/rbac/rbac_sql_zen_store.py +173 -0
- zenml/zen_server/routers/auth_endpoints.py +22 -11
- zenml/zen_server/routers/steps_endpoints.py +7 -1
- zenml/zen_server/template_execution/utils.py +3 -1
- zenml/zen_server/utils.py +4 -3
- zenml/zen_stores/base_zen_store.py +10 -2
- zenml/zen_stores/migrations/versions/0.71.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/26351d482b9e_add_step_run_unique_constraint.py +37 -0
- zenml/zen_stores/migrations/versions/a1237ba94fd8_add_model_version_producer_run_unique_.py +68 -0
- zenml/zen_stores/rest_zen_store.py +76 -43
- zenml/zen_stores/schemas/model_schemas.py +42 -6
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +7 -7
- zenml/zen_stores/schemas/pipeline_run_schemas.py +12 -6
- zenml/zen_stores/schemas/pipeline_schemas.py +5 -0
- zenml/zen_stores/schemas/step_run_schemas.py +8 -1
- zenml/zen_stores/sql_zen_store.py +332 -100
- {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241220.dist-info}/METADATA +5 -5
- {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241220.dist-info}/RECORD +179 -164
- zenml/zen_server/dashboard/assets/RunSelector-DkPiIiNr.js +0 -1
- zenml/zen_server/dashboard/assets/StackList-WvuKQusZ.js +0 -1
- zenml/zen_server/dashboard/assets/delete-run-CJdh1P_h.js +0 -1
- zenml/zen_server/dashboard/assets/index-DlGvJQPn.css +0 -1
- zenml/zen_server/dashboard/assets/page-0JE_-Ec1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BRLpxOt0.js +0 -1
- zenml/zen_server/dashboard/assets/page-BU7huvKw.js +0 -6
- zenml/zen_server/dashboard/assets/page-BvqLv2Ky.js +0 -1
- zenml/zen_server/dashboard/assets/page-CwxrFarU.js +0 -1
- zenml/zen_server/dashboard/assets/page-DfbXf_8s.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dnovpa0i.js +0 -3
- zenml/zen_server/dashboard/assets/page-Dot3LPmL.js +0 -1
- zenml/zen_server/dashboard/assets/page-Xynx4btY.js +0 -14
- zenml/zen_server/dashboard/assets/page-YpKAqVSa.js +0 -1
- {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241220.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241220.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.70.0.dev20241201.dist-info → zenml_nightly-0.71.0.dev20241220.dist-info}/entry_points.txt +0 -0
zenml/client.py
CHANGED
@@ -1702,6 +1702,7 @@ class Client(metaclass=ClientMetaClass):
|
|
1702
1702
|
updated: Optional[datetime] = None,
|
1703
1703
|
type: Optional[str] = None,
|
1704
1704
|
flavor: Optional[str] = None,
|
1705
|
+
user: Optional[Union[UUID, str]] = None,
|
1705
1706
|
workspace_id: Optional[Union[str, UUID]] = None,
|
1706
1707
|
user_id: Optional[Union[str, UUID]] = None,
|
1707
1708
|
hydrate: bool = False,
|
@@ -1727,6 +1728,7 @@ class Client(metaclass=ClientMetaClass):
|
|
1727
1728
|
flavor: Use the service flavor for filtering
|
1728
1729
|
workspace_id: The id of the workspace to filter by.
|
1729
1730
|
user_id: The id of the user to filter by.
|
1731
|
+
user: Filter by user name/ID.
|
1730
1732
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
1731
1733
|
by including metadata fields in the response.
|
1732
1734
|
running: Use the running status for filtering
|
@@ -1753,6 +1755,7 @@ class Client(metaclass=ClientMetaClass):
|
|
1753
1755
|
flavor=flavor,
|
1754
1756
|
workspace_id=workspace_id,
|
1755
1757
|
user_id=user_id,
|
1758
|
+
user=user,
|
1756
1759
|
running=running,
|
1757
1760
|
name=service_name,
|
1758
1761
|
pipeline_name=pipeline_name,
|
@@ -2249,6 +2252,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2249
2252
|
type: Optional[str] = None,
|
2250
2253
|
integration: Optional[str] = None,
|
2251
2254
|
user_id: Optional[Union[str, UUID]] = None,
|
2255
|
+
user: Optional[Union[UUID, str]] = None,
|
2252
2256
|
hydrate: bool = False,
|
2253
2257
|
) -> Page[FlavorResponse]:
|
2254
2258
|
"""Fetches all the flavor models.
|
@@ -2262,6 +2266,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2262
2266
|
created: Use to flavors by time of creation
|
2263
2267
|
updated: Use the last updated date for filtering
|
2264
2268
|
user_id: The id of the user to filter by.
|
2269
|
+
user: Filter by user name/ID.
|
2265
2270
|
name: The name of the flavor to filter by.
|
2266
2271
|
type: The type of the flavor to filter by.
|
2267
2272
|
integration: The integration of the flavor to filter by.
|
@@ -2277,6 +2282,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2277
2282
|
sort_by=sort_by,
|
2278
2283
|
logical_operator=logical_operator,
|
2279
2284
|
user_id=user_id,
|
2285
|
+
user=user,
|
2280
2286
|
name=name,
|
2281
2287
|
type=type,
|
2282
2288
|
integration=integration,
|
@@ -2661,13 +2667,16 @@ class Client(metaclass=ClientMetaClass):
|
|
2661
2667
|
updated: Optional[Union[datetime, str]] = None,
|
2662
2668
|
workspace_id: Optional[Union[str, UUID]] = None,
|
2663
2669
|
user_id: Optional[Union[str, UUID]] = None,
|
2670
|
+
user: Optional[Union[UUID, str]] = None,
|
2664
2671
|
pipeline_id: Optional[Union[str, UUID]] = None,
|
2665
2672
|
stack_id: Optional[Union[str, UUID]] = None,
|
2673
|
+
container_registry_id: Optional[Union[UUID, str]] = None,
|
2666
2674
|
is_local: Optional[bool] = None,
|
2667
2675
|
contains_code: Optional[bool] = None,
|
2668
2676
|
zenml_version: Optional[str] = None,
|
2669
2677
|
python_version: Optional[str] = None,
|
2670
2678
|
checksum: Optional[str] = None,
|
2679
|
+
stack_checksum: Optional[str] = None,
|
2671
2680
|
hydrate: bool = False,
|
2672
2681
|
) -> Page[PipelineBuildResponse]:
|
2673
2682
|
"""List all builds.
|
@@ -2682,13 +2691,17 @@ class Client(metaclass=ClientMetaClass):
|
|
2682
2691
|
updated: Use the last updated date for filtering
|
2683
2692
|
workspace_id: The id of the workspace to filter by.
|
2684
2693
|
user_id: The id of the user to filter by.
|
2694
|
+
user: Filter by user name/ID.
|
2685
2695
|
pipeline_id: The id of the pipeline to filter by.
|
2686
2696
|
stack_id: The id of the stack to filter by.
|
2697
|
+
container_registry_id: The id of the container registry to
|
2698
|
+
filter by.
|
2687
2699
|
is_local: Use to filter local builds.
|
2688
2700
|
contains_code: Use to filter builds that contain code.
|
2689
2701
|
zenml_version: The version of ZenML to filter by.
|
2690
2702
|
python_version: The Python version to filter by.
|
2691
2703
|
checksum: The build checksum to filter by.
|
2704
|
+
stack_checksum: The stack checksum to filter by.
|
2692
2705
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
2693
2706
|
by including metadata fields in the response.
|
2694
2707
|
|
@@ -2705,13 +2718,16 @@ class Client(metaclass=ClientMetaClass):
|
|
2705
2718
|
updated=updated,
|
2706
2719
|
workspace_id=workspace_id,
|
2707
2720
|
user_id=user_id,
|
2721
|
+
user=user,
|
2708
2722
|
pipeline_id=pipeline_id,
|
2709
2723
|
stack_id=stack_id,
|
2724
|
+
container_registry_id=container_registry_id,
|
2710
2725
|
is_local=is_local,
|
2711
2726
|
contains_code=contains_code,
|
2712
2727
|
zenml_version=zenml_version,
|
2713
2728
|
python_version=python_version,
|
2714
2729
|
checksum=checksum,
|
2730
|
+
stack_checksum=stack_checksum,
|
2715
2731
|
)
|
2716
2732
|
build_filter_model.set_scope_workspace(self.active_workspace.id)
|
2717
2733
|
return self.zen_store.list_builds(
|
@@ -2771,7 +2787,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2771
2787
|
allow_name_prefix_match: bool = True,
|
2772
2788
|
hydrate: bool = True,
|
2773
2789
|
) -> EventSourceResponse:
|
2774
|
-
"""Get
|
2790
|
+
"""Get an event source by name, ID or prefix.
|
2775
2791
|
|
2776
2792
|
Args:
|
2777
2793
|
name_id_or_prefix: The name, ID or prefix of the stack.
|
@@ -2804,6 +2820,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2804
2820
|
event_source_type: Optional[str] = None,
|
2805
2821
|
workspace_id: Optional[Union[str, UUID]] = None,
|
2806
2822
|
user_id: Optional[Union[str, UUID]] = None,
|
2823
|
+
user: Optional[Union[UUID, str]] = None,
|
2807
2824
|
hydrate: bool = False,
|
2808
2825
|
) -> Page[EventSourceResponse]:
|
2809
2826
|
"""Lists all event_sources.
|
@@ -2818,6 +2835,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2818
2835
|
updated: Use the last updated date for filtering
|
2819
2836
|
workspace_id: The id of the workspace to filter by.
|
2820
2837
|
user_id: The id of the user to filter by.
|
2838
|
+
user: Filter by user name/ID.
|
2821
2839
|
name: The name of the event_source to filter by.
|
2822
2840
|
flavor: The flavor of the event_source to filter by.
|
2823
2841
|
event_source_type: The subtype of the event_source to filter by.
|
@@ -2834,6 +2852,7 @@ class Client(metaclass=ClientMetaClass):
|
|
2834
2852
|
logical_operator=logical_operator,
|
2835
2853
|
workspace_id=workspace_id,
|
2836
2854
|
user_id=user_id,
|
2855
|
+
user=user,
|
2837
2856
|
name=name,
|
2838
2857
|
flavor=flavor,
|
2839
2858
|
plugin_subtype=event_source_type,
|
@@ -3001,6 +3020,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3001
3020
|
action_type: Optional[str] = None,
|
3002
3021
|
workspace_id: Optional[Union[str, UUID]] = None,
|
3003
3022
|
user_id: Optional[Union[str, UUID]] = None,
|
3023
|
+
user: Optional[Union[UUID, str]] = None,
|
3004
3024
|
hydrate: bool = False,
|
3005
3025
|
) -> Page[ActionResponse]:
|
3006
3026
|
"""List actions.
|
@@ -3015,6 +3035,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3015
3035
|
updated: Use the last updated date for filtering
|
3016
3036
|
workspace_id: The id of the workspace to filter by.
|
3017
3037
|
user_id: The id of the user to filter by.
|
3038
|
+
user: Filter by user name/ID.
|
3018
3039
|
name: The name of the action to filter by.
|
3019
3040
|
flavor: The flavor of the action to filter by.
|
3020
3041
|
action_type: The type of the action to filter by.
|
@@ -3031,6 +3052,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3031
3052
|
logical_operator=logical_operator,
|
3032
3053
|
workspace_id=workspace_id,
|
3033
3054
|
user_id=user_id,
|
3055
|
+
user=user,
|
3034
3056
|
name=name,
|
3035
3057
|
id=id,
|
3036
3058
|
flavor=flavor,
|
@@ -3179,6 +3201,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3179
3201
|
action_subtype: Optional[str] = None,
|
3180
3202
|
workspace_id: Optional[Union[str, UUID]] = None,
|
3181
3203
|
user_id: Optional[Union[str, UUID]] = None,
|
3204
|
+
user: Optional[Union[UUID, str]] = None,
|
3182
3205
|
hydrate: bool = False,
|
3183
3206
|
) -> Page[TriggerResponse]:
|
3184
3207
|
"""Lists all triggers.
|
@@ -3193,6 +3216,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3193
3216
|
updated: Use the last updated date for filtering
|
3194
3217
|
workspace_id: The id of the workspace to filter by.
|
3195
3218
|
user_id: The id of the user to filter by.
|
3219
|
+
user: Filter by user name/ID.
|
3196
3220
|
name: The name of the trigger to filter by.
|
3197
3221
|
event_source_id: The event source associated with the trigger.
|
3198
3222
|
action_id: The action associated with the trigger.
|
@@ -3215,6 +3239,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3215
3239
|
logical_operator=logical_operator,
|
3216
3240
|
workspace_id=workspace_id,
|
3217
3241
|
user_id=user_id,
|
3242
|
+
user=user,
|
3218
3243
|
name=name,
|
3219
3244
|
event_source_id=event_source_id,
|
3220
3245
|
action_id=action_id,
|
@@ -3365,6 +3390,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3365
3390
|
updated: Optional[Union[datetime, str]] = None,
|
3366
3391
|
workspace_id: Optional[Union[str, UUID]] = None,
|
3367
3392
|
user_id: Optional[Union[str, UUID]] = None,
|
3393
|
+
user: Optional[Union[UUID, str]] = None,
|
3368
3394
|
pipeline_id: Optional[Union[str, UUID]] = None,
|
3369
3395
|
stack_id: Optional[Union[str, UUID]] = None,
|
3370
3396
|
build_id: Optional[Union[str, UUID]] = None,
|
@@ -3383,6 +3409,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3383
3409
|
updated: Use the last updated date for filtering
|
3384
3410
|
workspace_id: The id of the workspace to filter by.
|
3385
3411
|
user_id: The id of the user to filter by.
|
3412
|
+
user: Filter by user name/ID.
|
3386
3413
|
pipeline_id: The id of the pipeline to filter by.
|
3387
3414
|
stack_id: The id of the stack to filter by.
|
3388
3415
|
build_id: The id of the build to filter by.
|
@@ -3403,6 +3430,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3403
3430
|
updated=updated,
|
3404
3431
|
workspace_id=workspace_id,
|
3405
3432
|
user_id=user_id,
|
3433
|
+
user=user,
|
3406
3434
|
pipeline_id=pipeline_id,
|
3407
3435
|
stack_id=stack_id,
|
3408
3436
|
build_id=build_id,
|
@@ -3488,6 +3516,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3488
3516
|
logical_operator: LogicalOperators = LogicalOperators.AND,
|
3489
3517
|
created: Optional[Union[datetime, str]] = None,
|
3490
3518
|
updated: Optional[Union[datetime, str]] = None,
|
3519
|
+
id: Optional[Union[UUID, str]] = None,
|
3491
3520
|
name: Optional[str] = None,
|
3492
3521
|
tag: Optional[str] = None,
|
3493
3522
|
workspace_id: Optional[Union[str, UUID]] = None,
|
@@ -3510,6 +3539,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3510
3539
|
logical_operator: Which logical operator to use [and, or].
|
3511
3540
|
created: Filter by the creation date.
|
3512
3541
|
updated: Filter by the last updated date.
|
3542
|
+
id: Filter by run template ID.
|
3513
3543
|
name: Filter by run template name.
|
3514
3544
|
tag: Filter by run template tags.
|
3515
3545
|
workspace_id: Filter by workspace ID.
|
@@ -3534,6 +3564,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3534
3564
|
logical_operator=logical_operator,
|
3535
3565
|
created=created,
|
3536
3566
|
updated=updated,
|
3567
|
+
id=id,
|
3537
3568
|
name=name,
|
3538
3569
|
tag=tag,
|
3539
3570
|
workspace_id=workspace_id,
|
@@ -3650,6 +3681,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3650
3681
|
name: Optional[str] = None,
|
3651
3682
|
workspace_id: Optional[Union[str, UUID]] = None,
|
3652
3683
|
user_id: Optional[Union[str, UUID]] = None,
|
3684
|
+
user: Optional[Union[UUID, str]] = None,
|
3653
3685
|
pipeline_id: Optional[Union[str, UUID]] = None,
|
3654
3686
|
orchestrator_id: Optional[Union[str, UUID]] = None,
|
3655
3687
|
active: Optional[Union[str, bool]] = None,
|
@@ -3674,6 +3706,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3674
3706
|
name: The name of the stack to filter by.
|
3675
3707
|
workspace_id: The id of the workspace to filter by.
|
3676
3708
|
user_id: The id of the user to filter by.
|
3709
|
+
user: Filter by user name/ID.
|
3677
3710
|
pipeline_id: The id of the pipeline to filter by.
|
3678
3711
|
orchestrator_id: The id of the orchestrator to filter by.
|
3679
3712
|
active: Use to filter by active status.
|
@@ -3700,6 +3733,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3700
3733
|
name=name,
|
3701
3734
|
workspace_id=workspace_id,
|
3702
3735
|
user_id=user_id,
|
3736
|
+
user=user,
|
3703
3737
|
pipeline_id=pipeline_id,
|
3704
3738
|
orchestrator_id=orchestrator_id,
|
3705
3739
|
active=active,
|
@@ -3940,6 +3974,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3940
3974
|
original_step_run_id: Optional[Union[str, UUID]] = None,
|
3941
3975
|
workspace_id: Optional[Union[str, UUID]] = None,
|
3942
3976
|
user_id: Optional[Union[str, UUID]] = None,
|
3977
|
+
user: Optional[Union[UUID, str]] = None,
|
3943
3978
|
model_version_id: Optional[Union[str, UUID]] = None,
|
3944
3979
|
model: Optional[Union[UUID, str]] = None,
|
3945
3980
|
hydrate: bool = False,
|
@@ -3958,6 +3993,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3958
3993
|
end_time: Use to filter by the time when the step finished running
|
3959
3994
|
workspace_id: The id of the workspace to filter by.
|
3960
3995
|
user_id: The id of the user to filter by.
|
3996
|
+
user: Filter by user name/ID.
|
3961
3997
|
pipeline_run_id: The id of the pipeline run to filter by.
|
3962
3998
|
deployment_id: The id of the deployment to filter by.
|
3963
3999
|
original_step_run_id: The id of the original step run to filter by.
|
@@ -3992,6 +4028,7 @@ class Client(metaclass=ClientMetaClass):
|
|
3992
4028
|
name=name,
|
3993
4029
|
workspace_id=workspace_id,
|
3994
4030
|
user_id=user_id,
|
4031
|
+
user=user,
|
3995
4032
|
model_version_id=model_version_id,
|
3996
4033
|
model=model,
|
3997
4034
|
)
|
@@ -4664,6 +4701,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4664
4701
|
scope: Optional[SecretScope] = None,
|
4665
4702
|
workspace_id: Optional[Union[str, UUID]] = None,
|
4666
4703
|
user_id: Optional[Union[str, UUID]] = None,
|
4704
|
+
user: Optional[Union[UUID, str]] = None,
|
4667
4705
|
hydrate: bool = False,
|
4668
4706
|
) -> Page[SecretResponse]:
|
4669
4707
|
"""Fetches all the secret models.
|
@@ -4683,6 +4721,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4683
4721
|
scope: The scope of the secret to filter by.
|
4684
4722
|
workspace_id: The id of the workspace to filter by.
|
4685
4723
|
user_id: The id of the user to filter by.
|
4724
|
+
user: Filter by user name/ID.
|
4686
4725
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
4687
4726
|
by including metadata fields in the response.
|
4688
4727
|
|
@@ -4699,6 +4738,7 @@ class Client(metaclass=ClientMetaClass):
|
|
4699
4738
|
sort_by=sort_by,
|
4700
4739
|
logical_operator=logical_operator,
|
4701
4740
|
user_id=user_id,
|
4741
|
+
user=user,
|
4702
4742
|
workspace_id=workspace_id,
|
4703
4743
|
name=name,
|
4704
4744
|
scope=scope,
|
@@ -5013,6 +5053,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5013
5053
|
name: Optional[str] = None,
|
5014
5054
|
workspace_id: Optional[Union[str, UUID]] = None,
|
5015
5055
|
user_id: Optional[Union[str, UUID]] = None,
|
5056
|
+
user: Optional[Union[UUID, str]] = None,
|
5016
5057
|
hydrate: bool = False,
|
5017
5058
|
) -> Page[CodeRepositoryResponse]:
|
5018
5059
|
"""List all code repositories.
|
@@ -5028,6 +5069,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5028
5069
|
name: The name of the code repository to filter by.
|
5029
5070
|
workspace_id: The id of the workspace to filter by.
|
5030
5071
|
user_id: The id of the user to filter by.
|
5072
|
+
user: Filter by user name/ID.
|
5031
5073
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
5032
5074
|
by including metadata fields in the response.
|
5033
5075
|
|
@@ -5045,6 +5087,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5045
5087
|
name=name,
|
5046
5088
|
workspace_id=workspace_id,
|
5047
5089
|
user_id=user_id,
|
5090
|
+
user=user,
|
5048
5091
|
)
|
5049
5092
|
filter_model.set_scope_workspace(self.active_workspace.id)
|
5050
5093
|
return self.zen_store.list_code_repositories(
|
@@ -5405,6 +5448,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5405
5448
|
resource_id: Optional[str] = None,
|
5406
5449
|
workspace_id: Optional[Union[str, UUID]] = None,
|
5407
5450
|
user_id: Optional[Union[str, UUID]] = None,
|
5451
|
+
user: Optional[Union[UUID, str]] = None,
|
5408
5452
|
labels: Optional[Dict[str, Optional[str]]] = None,
|
5409
5453
|
secret_id: Optional[Union[str, UUID]] = None,
|
5410
5454
|
hydrate: bool = False,
|
@@ -5427,6 +5471,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5427
5471
|
they can give access to.
|
5428
5472
|
workspace_id: The id of the workspace to filter by.
|
5429
5473
|
user_id: The id of the user to filter by.
|
5474
|
+
user: Filter by user name/ID.
|
5430
5475
|
name: The name of the service connector to filter by.
|
5431
5476
|
labels: The labels of the service connector to filter by.
|
5432
5477
|
secret_id: Filter by the id of the secret that is referenced by the
|
@@ -5444,6 +5489,7 @@ class Client(metaclass=ClientMetaClass):
|
|
5444
5489
|
logical_operator=logical_operator,
|
5445
5490
|
workspace_id=workspace_id or self.active_workspace.id,
|
5446
5491
|
user_id=user_id,
|
5492
|
+
user=user,
|
5447
5493
|
name=name,
|
5448
5494
|
connector_type=connector_type,
|
5449
5495
|
auth_method=auth_method,
|
@@ -6596,6 +6642,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6596
6642
|
client_id: Union[UUID, str, None] = None,
|
6597
6643
|
status: Union[OAuthDeviceStatus, str, None] = None,
|
6598
6644
|
trusted_device: Union[bool, str, None] = None,
|
6645
|
+
user: Optional[Union[UUID, str]] = None,
|
6599
6646
|
failed_auth_attempts: Union[int, str, None] = None,
|
6600
6647
|
last_login: Optional[Union[datetime, str, None]] = None,
|
6601
6648
|
hydrate: bool = False,
|
@@ -6613,6 +6660,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6613
6660
|
expires: Use the expiration date for filtering.
|
6614
6661
|
client_id: Use the client id for filtering.
|
6615
6662
|
status: Use the status for filtering.
|
6663
|
+
user: Filter by user name/ID.
|
6616
6664
|
trusted_device: Use the trusted device flag for filtering.
|
6617
6665
|
failed_auth_attempts: Use the failed auth attempts for filtering.
|
6618
6666
|
last_login: Use the last login date for filtering.
|
@@ -6632,6 +6680,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6632
6680
|
updated=updated,
|
6633
6681
|
expires=expires,
|
6634
6682
|
client_id=client_id,
|
6683
|
+
user=user,
|
6635
6684
|
status=status,
|
6636
6685
|
trusted_device=trusted_device,
|
6637
6686
|
failed_auth_attempts=failed_auth_attempts,
|
@@ -6730,7 +6779,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6730
6779
|
trigger_execution_id: UUID,
|
6731
6780
|
hydrate: bool = True,
|
6732
6781
|
) -> TriggerExecutionResponse:
|
6733
|
-
"""Get
|
6782
|
+
"""Get a trigger execution by ID.
|
6734
6783
|
|
6735
6784
|
Args:
|
6736
6785
|
trigger_execution_id: The ID of the trigger execution to get.
|
@@ -6751,6 +6800,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6751
6800
|
size: int = PAGE_SIZE_DEFAULT,
|
6752
6801
|
logical_operator: LogicalOperators = LogicalOperators.AND,
|
6753
6802
|
trigger_id: Optional[UUID] = None,
|
6803
|
+
user: Optional[Union[UUID, str]] = None,
|
6754
6804
|
hydrate: bool = False,
|
6755
6805
|
) -> Page[TriggerExecutionResponse]:
|
6756
6806
|
"""List all trigger executions matching the given filter criteria.
|
@@ -6761,6 +6811,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6761
6811
|
size: The maximum size of all pages.
|
6762
6812
|
logical_operator: Which logical operator to use [and, or].
|
6763
6813
|
trigger_id: ID of the trigger to filter by.
|
6814
|
+
user: Filter by user name/ID.
|
6764
6815
|
hydrate: Flag deciding whether to hydrate the output model(s)
|
6765
6816
|
by including metadata fields in the response.
|
6766
6817
|
|
@@ -6772,6 +6823,7 @@ class Client(metaclass=ClientMetaClass):
|
|
6772
6823
|
sort_by=sort_by,
|
6773
6824
|
page=page,
|
6774
6825
|
size=size,
|
6826
|
+
user=user,
|
6775
6827
|
logical_operator=logical_operator,
|
6776
6828
|
)
|
6777
6829
|
filter_model.set_scope_workspace(self.active_workspace.id)
|
@@ -66,7 +66,7 @@ class SecretReferenceMixin(BaseModel):
|
|
66
66
|
"but future versions of ZenML will require you to pass "
|
67
67
|
"in sensitive information as secrets. Check out the "
|
68
68
|
"documentation on how to configure values with secrets "
|
69
|
-
"here: https://docs.zenml.io/getting-started/deploying-zenml/
|
69
|
+
"here: https://docs.zenml.io/getting-started/deploying-zenml/secret-management"
|
70
70
|
)
|
71
71
|
continue
|
72
72
|
|
zenml/config/server_config.py
CHANGED
@@ -27,6 +27,7 @@ from zenml.constants import (
|
|
27
27
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_POLLING,
|
28
28
|
DEFAULT_ZENML_SERVER_DEVICE_AUTH_TIMEOUT,
|
29
29
|
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME,
|
30
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_MAX_LIFETIME,
|
30
31
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY,
|
31
32
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE,
|
32
33
|
DEFAULT_ZENML_SERVER_MAX_DEVICE_AUTH_ATTEMPTS,
|
@@ -269,6 +270,9 @@ class ServerConfiguration(BaseModel):
|
|
269
270
|
generic_api_token_lifetime: PositiveInt = (
|
270
271
|
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME
|
271
272
|
)
|
273
|
+
generic_api_token_max_lifetime: PositiveInt = (
|
274
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_MAX_LIFETIME
|
275
|
+
)
|
272
276
|
|
273
277
|
external_login_url: Optional[str] = None
|
274
278
|
external_user_info_url: Optional[str] = None
|
zenml/constants.py
CHANGED
@@ -170,6 +170,9 @@ ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
|
|
170
170
|
ENV_ZENML_IGNORE_FAILURE_HOOK = "ZENML_IGNORE_FAILURE_HOOK"
|
171
171
|
ENV_ZENML_CUSTOM_SOURCE_ROOT = "ZENML_CUSTOM_SOURCE_ROOT"
|
172
172
|
ENV_ZENML_WHEEL_PACKAGE_NAME = "ZENML_WHEEL_PACKAGE_NAME"
|
173
|
+
ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION = (
|
174
|
+
"ZENML_PIPELINE_API_TOKEN_EXPIRATION"
|
175
|
+
)
|
173
176
|
|
174
177
|
# ZenML Server environment variables
|
175
178
|
ENV_ZENML_SERVER_PREFIX = "ZENML_SERVER_"
|
@@ -268,6 +271,9 @@ DEFAULT_ZENML_SERVER_PIPELINE_RUN_AUTH_WINDOW = 60 * 48 # 48 hours
|
|
268
271
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_MINUTE = 5
|
269
272
|
DEFAULT_ZENML_SERVER_LOGIN_RATE_LIMIT_DAY = 1000
|
270
273
|
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_LIFETIME = 60 * 60 # 1 hour
|
274
|
+
DEFAULT_ZENML_SERVER_GENERIC_API_TOKEN_MAX_LIFETIME = (
|
275
|
+
60 * 60 * 24 * 7
|
276
|
+
) # 7 days
|
271
277
|
|
272
278
|
DEFAULT_ZENML_SERVER_SECURE_HEADERS_HSTS = (
|
273
279
|
"max-age=63072000; includeSubdomains"
|
@@ -466,3 +472,7 @@ BANNED_NAME_CHARACTERS = "\t\n\r\v\f"
|
|
466
472
|
|
467
473
|
|
468
474
|
STACK_DEPLOYMENT_API_TOKEN_EXPIRATION = 60 * 6 # 6 hours
|
475
|
+
|
476
|
+
ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION = handle_int_env_var(
|
477
|
+
ENV_ZENML_PIPELINE_RUN_API_TOKEN_EXPIRATION, default=0
|
478
|
+
)
|
@@ -25,6 +25,7 @@ from zenml.io import fileio
|
|
25
25
|
from zenml.logger import get_logger
|
26
26
|
from zenml.stack import Flavor, StackComponent
|
27
27
|
from zenml.stack.stack_component import StackComponentConfig
|
28
|
+
from zenml.utils.archivable import ArchiveType
|
28
29
|
|
29
30
|
if TYPE_CHECKING:
|
30
31
|
from zenml.container_registries import BaseContainerRegistry
|
@@ -100,6 +101,7 @@ class BaseImageBuilder(StackComponent, ABC):
|
|
100
101
|
def _upload_build_context(
|
101
102
|
build_context: "BuildContext",
|
102
103
|
parent_path_directory_name: str,
|
104
|
+
archive_type: ArchiveType = ArchiveType.TAR_GZ,
|
103
105
|
) -> str:
|
104
106
|
"""Uploads a Docker image build context to a remote location.
|
105
107
|
|
@@ -109,6 +111,7 @@ class BaseImageBuilder(StackComponent, ABC):
|
|
109
111
|
the build context to. It will be appended to the artifact
|
110
112
|
store path to create the parent path where the build context
|
111
113
|
will be uploaded to.
|
114
|
+
archive_type: The type of archive to create.
|
112
115
|
|
113
116
|
Returns:
|
114
117
|
The path to the uploaded build context.
|
@@ -119,7 +122,7 @@ class BaseImageBuilder(StackComponent, ABC):
|
|
119
122
|
|
120
123
|
hash_ = hashlib.sha1() # nosec
|
121
124
|
with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
|
122
|
-
build_context.write_archive(f,
|
125
|
+
build_context.write_archive(f, archive_type)
|
123
126
|
|
124
127
|
while True:
|
125
128
|
data = f.read(64 * 1024)
|
@@ -127,7 +130,7 @@ class BaseImageBuilder(StackComponent, ABC):
|
|
127
130
|
break
|
128
131
|
hash_.update(data)
|
129
132
|
|
130
|
-
filename = f"{hash_.hexdigest()}.
|
133
|
+
filename = f"{hash_.hexdigest()}.{archive_type.value}"
|
131
134
|
filepath = f"{parent_path}/{filename}"
|
132
135
|
if not fileio.exists(filepath):
|
133
136
|
logger.info("Uploading build context to `%s`.", filepath)
|
@@ -20,7 +20,7 @@ from zenml.constants import REPOSITORY_DIRECTORY_NAME
|
|
20
20
|
from zenml.io import fileio
|
21
21
|
from zenml.logger import get_logger
|
22
22
|
from zenml.utils import io_utils, string_utils
|
23
|
-
from zenml.utils.archivable import Archivable
|
23
|
+
from zenml.utils.archivable import Archivable, ArchiveType
|
24
24
|
|
25
25
|
logger = get_logger(__name__)
|
26
26
|
|
@@ -69,28 +69,19 @@ class BuildContext(Archivable):
|
|
69
69
|
return None
|
70
70
|
|
71
71
|
def write_archive(
|
72
|
-
self,
|
72
|
+
self,
|
73
|
+
output_file: IO[bytes],
|
74
|
+
archive_type: ArchiveType = ArchiveType.TAR_GZ,
|
73
75
|
) -> None:
|
74
76
|
"""Writes an archive of the build context to the given file.
|
75
77
|
|
76
78
|
Args:
|
77
79
|
output_file: The file to write the archive to.
|
78
|
-
|
80
|
+
archive_type: The type of archive to create.
|
79
81
|
"""
|
80
|
-
|
81
|
-
|
82
|
-
files = self.get_files()
|
83
|
-
extra_files = self.get_extra_files()
|
84
|
-
|
85
|
-
context_archive = docker_build_utils.create_archive(
|
86
|
-
fileobj=output_file,
|
87
|
-
root=self._root,
|
88
|
-
files=sorted(files.keys()),
|
89
|
-
gzip=use_gzip,
|
90
|
-
extra_files=list(extra_files.items()),
|
91
|
-
)
|
82
|
+
super().write_archive(output_file, archive_type)
|
92
83
|
|
93
|
-
build_context_size = os.path.getsize(
|
84
|
+
build_context_size = os.path.getsize(output_file.name)
|
94
85
|
if (
|
95
86
|
self._root
|
96
87
|
and build_context_size > 50 * 1024 * 1024
|
@@ -33,6 +33,7 @@ AWS_SAGEMAKER_ORCHESTRATOR_FLAVOR = "sagemaker"
|
|
33
33
|
AWS_CONNECTOR_TYPE = "aws"
|
34
34
|
AWS_RESOURCE_TYPE = "aws-generic"
|
35
35
|
S3_RESOURCE_TYPE = "s3-bucket"
|
36
|
+
AWS_IMAGE_BUILDER_FLAVOR = "aws"
|
36
37
|
|
37
38
|
class AWSIntegration(Integration):
|
38
39
|
"""Definition of AWS integration for ZenML."""
|
@@ -59,12 +60,14 @@ class AWSIntegration(Integration):
|
|
59
60
|
"""
|
60
61
|
from zenml.integrations.aws.flavors import (
|
61
62
|
AWSContainerRegistryFlavor,
|
63
|
+
AWSImageBuilderFlavor,
|
62
64
|
SagemakerOrchestratorFlavor,
|
63
65
|
SagemakerStepOperatorFlavor,
|
64
66
|
)
|
65
67
|
|
66
68
|
return [
|
67
69
|
AWSContainerRegistryFlavor,
|
70
|
+
AWSImageBuilderFlavor,
|
68
71
|
SagemakerStepOperatorFlavor,
|
69
72
|
SagemakerOrchestratorFlavor,
|
70
73
|
]
|
@@ -17,6 +17,10 @@ from zenml.integrations.aws.flavors.aws_container_registry_flavor import (
|
|
17
17
|
AWSContainerRegistryConfig,
|
18
18
|
AWSContainerRegistryFlavor,
|
19
19
|
)
|
20
|
+
from zenml.integrations.aws.flavors.aws_image_builder_flavor import (
|
21
|
+
AWSImageBuilderConfig,
|
22
|
+
AWSImageBuilderFlavor,
|
23
|
+
)
|
20
24
|
from zenml.integrations.aws.flavors.sagemaker_orchestrator_flavor import (
|
21
25
|
SagemakerOrchestratorConfig,
|
22
26
|
SagemakerOrchestratorFlavor,
|
@@ -29,6 +33,8 @@ from zenml.integrations.aws.flavors.sagemaker_step_operator_flavor import (
|
|
29
33
|
__all__ = [
|
30
34
|
"AWSContainerRegistryFlavor",
|
31
35
|
"AWSContainerRegistryConfig",
|
36
|
+
"AWSImageBuilderConfig",
|
37
|
+
"AWSImageBuilderFlavor",
|
32
38
|
"SagemakerStepOperatorFlavor",
|
33
39
|
"SagemakerStepOperatorConfig",
|
34
40
|
"SagemakerOrchestratorFlavor",
|