zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +2 -2
- RELEASE_NOTES.md +120 -0
- zenml/VERSION +1 -1
- zenml/__init__.py +0 -4
- zenml/actions/pipeline_run/pipeline_run_action.py +19 -17
- zenml/analytics/enums.py +4 -6
- zenml/cli/__init__.py +28 -76
- zenml/cli/base.py +2 -2
- zenml/cli/pipeline.py +54 -61
- zenml/cli/stack.py +6 -8
- zenml/cli/web_login.py +8 -0
- zenml/client.py +232 -103
- zenml/config/build_configuration.py +43 -17
- zenml/config/compiler.py +14 -22
- zenml/config/docker_settings.py +80 -57
- zenml/config/pipeline_run_configuration.py +3 -0
- zenml/config/server_config.py +3 -0
- zenml/config/source.py +60 -1
- zenml/constants.py +11 -2
- zenml/entrypoints/base_entrypoint_configuration.py +53 -8
- zenml/enums.py +4 -1
- zenml/environment.py +25 -9
- zenml/image_builders/base_image_builder.py +1 -1
- zenml/image_builders/build_context.py +25 -72
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +13 -4
- zenml/integrations/azure/__init__.py +4 -0
- zenml/integrations/azure/flavors/__init__.py +11 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +263 -0
- zenml/{_hub → integrations/azure/orchestrators}/__init__.py +7 -2
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +544 -0
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +86 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +3 -0
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +20 -2
- zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +19 -13
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +7 -2
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +123 -6
- zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +1 -1
- zenml/integrations/mlflow/__init__.py +1 -1
- zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +3 -1
- zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +3 -0
- zenml/logger.py +13 -0
- zenml/models/__init__.py +26 -22
- zenml/models/v2/base/filter.py +32 -0
- zenml/models/v2/core/pipeline.py +73 -89
- zenml/models/v2/core/pipeline_build.py +15 -11
- zenml/models/v2/core/pipeline_deployment.py +72 -24
- zenml/models/v2/core/pipeline_run.py +65 -1
- zenml/models/v2/core/run_template.py +393 -0
- zenml/models/v2/core/server_settings.py +12 -0
- zenml/models/v2/core/user.py +0 -21
- zenml/models/v2/misc/server_models.py +7 -1
- zenml/models/v2/misc/stack_deployment.py +5 -0
- zenml/models/v2/misc/user_auth.py +0 -7
- zenml/new/pipelines/build_utils.py +220 -89
- zenml/new/pipelines/code_archive.py +157 -0
- zenml/new/pipelines/pipeline.py +46 -78
- zenml/new/pipelines/run_utils.py +79 -1
- zenml/post_execution/pipeline.py +1 -4
- zenml/service_connectors/service_connector_utils.py +18 -2
- zenml/stack_deployments/aws_stack_deployment.py +32 -8
- zenml/stack_deployments/azure_stack_deployment.py +122 -10
- zenml/stack_deployments/gcp_stack_deployment.py +36 -7
- zenml/stack_deployments/stack_deployment.py +23 -7
- zenml/steps/base_step.py +3 -0
- zenml/steps/utils.py +0 -4
- zenml/utils/archivable.py +149 -0
- zenml/utils/code_utils.py +244 -0
- zenml/utils/notebook_utils.py +122 -0
- zenml/utils/package_utils.py +39 -0
- zenml/utils/pipeline_docker_image_builder.py +3 -96
- zenml/utils/source_utils.py +109 -1
- zenml/zen_server/dashboard/assets/{404-B_YdvmwS.js → 404-CRAA_Lew.js} +1 -1
- zenml/zen_server/dashboard/assets/@radix-BXWm7HOa.js +85 -0
- zenml/zen_server/dashboard/assets/{@react-router-CO-OsFwI.js → @react-router-l3lMcXA2.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-l_1hUr1S.js → @reactflow-CeVxyqYT.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-DYiOyJUL.js → @tanstack-FmcYZMuX.js} +4 -4
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +1 -0
- zenml/zen_server/dashboard/assets/{AwarenessChannel-CFg5iX4Z.js → AwarenessChannel-CLXo5rKM.js} +1 -1
- zenml/zen_server/dashboard/assets/{CodeSnippet-Dvkx_82E.js → CodeSnippet-D0VLxT2A.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +1 -0
- zenml/zen_server/dashboard/assets/{Commands-DoN1xrEq.js → Commands-JrcZK-3j.js} +1 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-Ck-nZ43m.js → CsvVizualization-D3kAypDj.js} +3 -3
- zenml/zen_server/dashboard/assets/DisplayDate-DizbSeT-.js +1 -0
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BMLnFVlB.js → EmptyState-BHblM39I.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-kLtljEOM.js → Error-C6LeJSER.js} +1 -1
- zenml/zen_server/dashboard/assets/{ExecutionStatus-DguLLgTK.js → ExecutionStatus-jH4OrWBq.js} +1 -1
- zenml/zen_server/dashboard/assets/{Helpbox-BXUMP21n.js → Helpbox-aAB2XP-z.js} +1 -1
- zenml/zen_server/dashboard/assets/{Infobox-DSt0O-dm.js → Infobox-BQ0aty32.js} +1 -1
- zenml/zen_server/dashboard/assets/{InlineAvatar-xsrsIGE-.js → InlineAvatar-DpTLgM3Q.js} +1 -1
- zenml/zen_server/dashboard/assets/Lock-CNyJvf2r.js +1 -0
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-xp3hhULl.js → MarkdownVisualization-Bajxn0HY.js} +1 -1
- zenml/zen_server/dashboard/assets/NumberBox-BmKE0qnO.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-DUveqlva.js → PasswordChecker-yGGoJSB-.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +1 -0
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +1 -0
- zenml/zen_server/dashboard/assets/{SetPassword-BXGTWiwj.js → SetPassword-52sNxNiO.js} +1 -1
- zenml/zen_server/dashboard/assets/{SuccessStep-DZC60t0x.js → SuccessStep-DlkItqYG.js} +1 -1
- zenml/zen_server/dashboard/assets/Tick-uxv80Q6a.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DGvwFWO1.js → UpdatePasswordSchemas-oN4G3sKz.js} +1 -1
- zenml/zen_server/dashboard/assets/{aws-BgKTfTfx.js → aws-0_3UsPif.js} +1 -1
- zenml/zen_server/dashboard/assets/{check-circle-i56092KI.js → check-circle-1_I207rW.js} +1 -1
- zenml/zen_server/dashboard/assets/chevron-down-BpaF8JqM.js +1 -0
- zenml/zen_server/dashboard/assets/{chevron-right-double-CZBOf6JM.js → chevron-right-double-Dk8e2L99.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-C_yFCAkP.js → cloud-only-BkUuI0lZ.js} +1 -1
- zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
- zenml/zen_server/dashboard/assets/{copy-BXNk6BjL.js → copy-f3XGPPxt.js} +1 -1
- zenml/zen_server/dashboard/assets/{database-1xWSgZfO.js → database-cXYNX9tt.js} +1 -1
- zenml/zen_server/dashboard/assets/{docker-CQMVm_4d.js → docker-8uj__HHK.js} +1 -1
- zenml/zen_server/dashboard/assets/dots-horizontal-sKQlWEni.js +1 -0
- zenml/zen_server/dashboard/assets/edit-C0MVvPD2.js +1 -0
- zenml/zen_server/dashboard/assets/{file-text-CqD_iu6l.js → file-text-B9JibxTs.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-bu_DgLKI.js → help-FuHlZwn0.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-rK_Wuy2W.js → index-Bd1xgUQG.js} +1 -1
- zenml/zen_server/dashboard/assets/index-DaGknux4.css +1 -0
- zenml/zen_server/dashboard/assets/{index-BczVOqUf.js → index-DhIZtpxB.js} +5 -5
- zenml/zen_server/dashboard/assets/index.esm-DT4uyn2i.js +1 -0
- zenml/zen_server/dashboard/assets/layout-D6oiSbfd.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-CrHrndTI.js → login-mutation-13A_JSVA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-D8k8BVFf.js → logs-CgeE2vZP.js} +1 -1
- zenml/zen_server/dashboard/assets/{not-found-DYa4pC-C.js → not-found-B0Mmb90p.js} +1 -1
- zenml/zen_server/dashboard/assets/package-DdkziX79.js +1 -0
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +1 -0
- zenml/zen_server/dashboard/assets/{page-MFQyIJd3.js → page-B3ozwdD1.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BkuQDIf-.js → page-BGwA9B1M.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-1iL8aMqs.js → page-BkjAUyTA.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +1 -0
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +2 -0
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +1 -0
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +9 -0
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +1 -0
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +1 -0
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +1 -0
- zenml/zen_server/dashboard/assets/{page-8a4UMKXZ.js → page-Cgn-6v2Y.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +1 -0
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +1 -0
- zenml/zen_server/dashboard/assets/page-DLpOnf7u.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BhgCDInH.js → page-DSTQnBk-.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-1h_sD1jz.js → page-DTysUGOy.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-2grKx_MY.js → page-D_EXUFJb.js} +1 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BDns21Iz.js → page-DugsjcQ_.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-C6-UGEbH.js → page-OFKSPyN7.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-BkeAAYwp.js → page-RnG-qhv9.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-CCNRIt_f.js → page-T2BtjwPl.js} +1 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +1 -0
- zenml/zen_server/dashboard/assets/{page-BnaevhnB.js → page-YiF_fNbe.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-uA5prJGY.js → page-hQaiQXfg.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-3-5nOJ6m.js +1 -0
- zenml/zen_server/dashboard/assets/{play-circle-CNtZKDnW.js → play-circle-XSkLR12B.js} +1 -1
- zenml/zen_server/dashboard/assets/plus-FB9-lEq_.js +1 -0
- zenml/zen_server/dashboard/assets/refresh-COb6KYDi.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +14 -0
- zenml/zen_server/dashboard/assets/{stack-detail-query-Cficsl6d.js → stack-detail-query-B-US_-wa.js} +1 -1
- zenml/zen_server/dashboard/assets/{terminal-By9cErXc.js → terminal-grtjrIEJ.js} +1 -1
- zenml/zen_server/dashboard/assets/trash-Cd5CSFqA.js +1 -0
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-7d8xi1tS.js → update-server-settings-mutation-B8GB_ubU.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-D7mAQGUM.js → url-hcMJkz8p.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-BhoGpZ63.js → zod-CnykDKJj.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.12246c7548e71e2c4438e496360de80c.js → precache-manifest.9c473c96a43298343a7ce1256183123b.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.3b27024b.chunk.js → main.463c90b9.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.3b27024b.chunk.js.map → main.463c90b9.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/rbac/models.py +1 -0
- zenml/zen_server/rbac/utils.py +4 -0
- zenml/zen_server/routers/pipeline_builds_endpoints.py +2 -66
- zenml/zen_server/routers/pipeline_deployments_endpoints.py +2 -53
- zenml/zen_server/routers/pipelines_endpoints.py +1 -74
- zenml/zen_server/routers/run_templates_endpoints.py +212 -0
- zenml/zen_server/routers/stack_deployment_endpoints.py +6 -0
- zenml/zen_server/routers/users_endpoints.py +0 -7
- zenml/zen_server/routers/workspaces_endpoints.py +79 -0
- zenml/zen_server/{pipeline_deployment → template_execution}/runner_entrypoint_configuration.py +1 -8
- zenml/zen_server/{pipeline_deployment → template_execution}/utils.py +214 -92
- zenml/zen_server/utils.py +77 -2
- zenml/zen_server/zen_server_api.py +54 -2
- zenml/zen_stores/base_zen_store.py +7 -1
- zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
- zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
- zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
- zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
- zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
- zenml/zen_stores/rest_zen_store.py +112 -39
- zenml/zen_stores/schemas/__init__.py +2 -0
- zenml/zen_stores/schemas/pipeline_build_schemas.py +3 -3
- zenml/zen_stores/schemas/pipeline_deployment_schemas.py +32 -2
- zenml/zen_stores/schemas/pipeline_run_schemas.py +29 -3
- zenml/zen_stores/schemas/pipeline_schemas.py +29 -30
- zenml/zen_stores/schemas/run_template_schemas.py +264 -0
- zenml/zen_stores/schemas/server_settings_schemas.py +2 -0
- zenml/zen_stores/schemas/step_run_schemas.py +11 -4
- zenml/zen_stores/schemas/user_schemas.py +0 -2
- zenml/zen_stores/sql_zen_store.py +389 -151
- zenml/zen_stores/template_utils.py +261 -0
- zenml/zen_stores/zen_store_interface.py +93 -20
- {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/METADATA +3 -3
- {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/RECORD +211 -184
- zenml/_hub/client.py +0 -289
- zenml/_hub/constants.py +0 -21
- zenml/_hub/utils.py +0 -79
- zenml/cli/hub.py +0 -1116
- zenml/models/v2/core/pipeline_namespace.py +0 -113
- zenml/models/v2/misc/hub_plugin_models.py +0 -79
- zenml/new/pipelines/deserialization_utils.py +0 -292
- zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +0 -85
- zenml/zen_server/dashboard/assets/CollapsibleCard-opiuBHHc.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Cr7xYEPb.js +0 -2
- zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +0 -1
- zenml/zen_server/dashboard/assets/Pagination-C6X-mifw.js +0 -1
- zenml/zen_server/dashboard/assets/index-EpMIKgrI.css +0 -1
- zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +0 -1
- zenml/zen_server/dashboard/assets/package-B3fWP-Dh.js +0 -1
- zenml/zen_server/dashboard/assets/page-5NCOHOsy.js +0 -1
- zenml/zen_server/dashboard/assets/page-B6h3iaHJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bi-wtWiO.js +0 -5
- zenml/zen_server/dashboard/assets/page-Bq0YxkLV.js +0 -1
- zenml/zen_server/dashboard/assets/page-Bs2F4eoD.js +0 -2
- zenml/zen_server/dashboard/assets/page-CHNxpz3n.js +0 -1
- zenml/zen_server/dashboard/assets/page-DgorQFqi.js +0 -1
- zenml/zen_server/dashboard/assets/page-K8ebxVIs.js +0 -1
- zenml/zen_server/dashboard/assets/page-TgCF0P_U.js +0 -1
- zenml/zen_server/dashboard/assets/page-ZnCEe-eK.js +0 -9
- zenml/zen_server/dashboard/assets/persist-D7HJNBWx.js +0 -1
- zenml/zen_server/dashboard/assets/plus-C8WOyCzt.js +0 -1
- zenml/zen_server/dashboard/assets/secrets-video-OBJ6irhH.svg +0 -21
- zenml/zen_server/dashboard/assets/stacks-video-7gfxpAq4.svg +0 -21
- /zenml/zen_server/{pipeline_deployment → template_execution}/__init__.py +0 -0
- /zenml/zen_server/{pipeline_deployment → template_execution}/workload_manager_interface.py +0 -0
- {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/entry_points.txt +0 -0
zenml/config/compiler.py
CHANGED
@@ -39,6 +39,7 @@ from zenml.config.step_configurations import (
|
|
39
39
|
from zenml.environment import get_run_environment_dict
|
40
40
|
from zenml.exceptions import StackValidationError
|
41
41
|
from zenml.models import PipelineDeploymentBase
|
42
|
+
from zenml.new.pipelines.run_utils import get_default_run_name
|
42
43
|
from zenml.utils import pydantic_utils, settings_utils
|
43
44
|
|
44
45
|
if TYPE_CHECKING:
|
@@ -74,7 +75,7 @@ class Compiler:
|
|
74
75
|
pipeline: "Pipeline",
|
75
76
|
stack: "Stack",
|
76
77
|
run_configuration: PipelineRunConfiguration,
|
77
|
-
) ->
|
78
|
+
) -> PipelineDeploymentBase:
|
78
79
|
"""Compiles a ZenML pipeline to a serializable representation.
|
79
80
|
|
80
81
|
Args:
|
@@ -83,7 +84,7 @@ class Compiler:
|
|
83
84
|
run_configuration: The run configuration for this pipeline.
|
84
85
|
|
85
86
|
Returns:
|
86
|
-
The compiled pipeline deployment
|
87
|
+
The compiled pipeline deployment.
|
87
88
|
"""
|
88
89
|
logger.debug("Compiling pipeline `%s`.", pipeline.name)
|
89
90
|
# Copy the pipeline before we apply any run-level configurations, so
|
@@ -127,12 +128,17 @@ class Compiler:
|
|
127
128
|
|
128
129
|
self._ensure_required_stack_components_exist(stack=stack, steps=steps)
|
129
130
|
|
130
|
-
run_name = run_configuration.run_name or
|
131
|
+
run_name = run_configuration.run_name or get_default_run_name(
|
131
132
|
pipeline_name=pipeline.name
|
132
133
|
)
|
133
134
|
|
134
135
|
client_version, server_version = get_zenml_versions()
|
135
136
|
|
137
|
+
step_specs = [step.spec for step in steps.values()]
|
138
|
+
pipeline_spec = self._compute_pipeline_spec(
|
139
|
+
pipeline=pipeline, step_specs=step_specs
|
140
|
+
)
|
141
|
+
|
136
142
|
deployment = PipelineDeploymentBase(
|
137
143
|
run_name_template=run_name,
|
138
144
|
pipeline_configuration=pipeline.configuration,
|
@@ -140,17 +146,15 @@ class Compiler:
|
|
140
146
|
client_environment=get_run_environment_dict(),
|
141
147
|
client_version=client_version,
|
142
148
|
server_version=server_version,
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
pipeline=pipeline, step_specs=step_specs
|
149
|
+
pipeline_version_hash=pipeline._compute_unique_identifier(
|
150
|
+
pipeline_spec=pipeline_spec
|
151
|
+
),
|
152
|
+
pipeline_spec=pipeline_spec,
|
148
153
|
)
|
149
154
|
|
150
155
|
logger.debug("Compiled pipeline deployment: %s", deployment)
|
151
|
-
logger.debug("Compiled pipeline spec: %s", pipeline_spec)
|
152
156
|
|
153
|
-
return deployment
|
157
|
+
return deployment
|
154
158
|
|
155
159
|
def compile_spec(self, pipeline: "Pipeline") -> PipelineSpec:
|
156
160
|
"""Compiles a ZenML pipeline to a pipeline spec.
|
@@ -474,18 +478,6 @@ class Compiler:
|
|
474
478
|
)
|
475
479
|
return Step(spec=step_spec, config=complete_step_configuration)
|
476
480
|
|
477
|
-
@staticmethod
|
478
|
-
def _get_default_run_name(pipeline_name: str) -> str:
|
479
|
-
"""Gets the default name for a pipeline run.
|
480
|
-
|
481
|
-
Args:
|
482
|
-
pipeline_name: Name of the pipeline which will be run.
|
483
|
-
|
484
|
-
Returns:
|
485
|
-
Run name.
|
486
|
-
"""
|
487
|
-
return f"{pipeline_name}-{{date}}-{{time}}"
|
488
|
-
|
489
481
|
def _get_sorted_invocations(
|
490
482
|
self,
|
491
483
|
pipeline: "Pipeline",
|
zenml/config/docker_settings.py
CHANGED
@@ -16,8 +16,7 @@
|
|
16
16
|
from enum import Enum
|
17
17
|
from typing import Any, Dict, List, Optional, Union
|
18
18
|
|
19
|
-
from pydantic import BaseModel, Field, model_validator
|
20
|
-
from pydantic_settings import SettingsConfigDict
|
19
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
21
20
|
|
22
21
|
from zenml.config.base_settings import BaseSettings
|
23
22
|
from zenml.logger import get_logger
|
@@ -49,15 +48,6 @@ class PythonEnvironmentExportMethod(Enum):
|
|
49
48
|
}[self]
|
50
49
|
|
51
50
|
|
52
|
-
class SourceFileMode(Enum):
|
53
|
-
"""Different methods to handle source files in Docker images."""
|
54
|
-
|
55
|
-
INCLUDE = "include"
|
56
|
-
DOWNLOAD_OR_INCLUDE = "download_or_include"
|
57
|
-
DOWNLOAD = "download"
|
58
|
-
IGNORE = "ignore"
|
59
|
-
|
60
|
-
|
61
51
|
class PythonPackageInstaller(Enum):
|
62
52
|
"""Different installers for python packages."""
|
63
53
|
|
@@ -102,7 +92,6 @@ class DockerSettings(BaseSettings):
|
|
102
92
|
Depending on the configuration of this object, requirements will be
|
103
93
|
installed in the following order (each step optional):
|
104
94
|
- The packages installed in your local python environment
|
105
|
-
- The packages specified via the `required_hub_plugins` attribute
|
106
95
|
- The packages required by the stack unless this is disabled by setting
|
107
96
|
`install_stack_requirements=False`.
|
108
97
|
- The packages specified via the `required_integrations`
|
@@ -135,10 +124,9 @@ class DockerSettings(BaseSettings):
|
|
135
124
|
when the `dockerfile` attribute is set. If this is left empty, the
|
136
125
|
build context will only contain the Dockerfile.
|
137
126
|
parent_image_build_config: Configuration for the parent image build.
|
138
|
-
build_options: DEPRECATED, use parent_image_build_config.build_options
|
139
|
-
instead.
|
140
127
|
skip_build: If set to `True`, the parent image will be used directly to
|
141
128
|
run the steps of your pipeline.
|
129
|
+
prevent_build_reuse: Prevent the reuse of an existing build.
|
142
130
|
target_repository: Name of the Docker repository to which the
|
143
131
|
image should be pushed. This repository will be appended to the
|
144
132
|
registry URI of the container registry of your stack and should
|
@@ -161,11 +149,7 @@ class DockerSettings(BaseSettings):
|
|
161
149
|
required_integrations: List of ZenML integrations that should be
|
162
150
|
installed. All requirements for the specified integrations will
|
163
151
|
be installed inside the Docker image.
|
164
|
-
required_hub_plugins:
|
165
|
-
Expected format: '(<author_username>/)<plugin_name>==<version>'.
|
166
|
-
If no version is specified, the latest version is taken. The
|
167
|
-
packages of required plugins and all their dependencies will be
|
168
|
-
installed inside the Docker image.
|
152
|
+
required_hub_plugins: DEPRECATED/UNUSED.
|
169
153
|
install_stack_requirements: If `True`, ZenML will automatically detect
|
170
154
|
if components of your active stack are part of a ZenML integration
|
171
155
|
and install the corresponding requirements and apt packages.
|
@@ -176,33 +160,32 @@ class DockerSettings(BaseSettings):
|
|
176
160
|
environment: Dictionary of environment variables to set inside the
|
177
161
|
Docker image.
|
178
162
|
build_config: Configuration for the main image build.
|
179
|
-
dockerignore: DEPRECATED, use build_config.dockerignore instead.
|
180
|
-
copy_files: DEPRECATED, use the `source_files` attribute instead.
|
181
|
-
copy_global_config: DEPRECATED/UNUSED.
|
182
163
|
user: If not `None`, will set the user, make it owner of the `/app`
|
183
164
|
directory which contains all the user code and run the container
|
184
165
|
entrypoint as this user.
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
166
|
+
allow_including_files_in_images: If `True`, code can be included in the
|
167
|
+
Docker images if code download from a code repository or artifact
|
168
|
+
store is disabled or not possible.
|
169
|
+
allow_download_from_code_repository: If `True`, code can be downloaded
|
170
|
+
from a code repository if possible.
|
171
|
+
allow_download_from_artifact_store: If `True`, code can be downloaded
|
172
|
+
from the artifact store.
|
173
|
+
build_options: DEPRECATED, use parent_image_build_config.build_options
|
174
|
+
instead.
|
175
|
+
dockerignore: DEPRECATED, use build_config.dockerignore instead.
|
176
|
+
copy_files: DEPRECATED/UNUSED.
|
177
|
+
copy_global_config: DEPRECATED/UNUSED.
|
178
|
+
source_files: DEPRECATED. Use allow_including_files_in_images,
|
179
|
+
allow_download_from_code_repository and
|
180
|
+
allow_download_from_artifact_store instead.
|
198
181
|
"""
|
199
182
|
|
200
183
|
parent_image: Optional[str] = None
|
201
184
|
dockerfile: Optional[str] = None
|
202
185
|
build_context_root: Optional[str] = None
|
203
|
-
build_options: Dict[str, Any] = {}
|
204
186
|
parent_image_build_config: Optional[DockerBuildConfig] = None
|
205
187
|
skip_build: bool = False
|
188
|
+
prevent_build_reuse: bool = False
|
206
189
|
target_repository: Optional[str] = None
|
207
190
|
python_package_installer: PythonPackageInstaller = (
|
208
191
|
PythonPackageInstaller.PIP
|
@@ -215,49 +198,89 @@ class DockerSettings(BaseSettings):
|
|
215
198
|
default=None, union_mode="left_to_right"
|
216
199
|
)
|
217
200
|
required_integrations: List[str] = []
|
218
|
-
required_hub_plugins: List[str] = []
|
219
201
|
install_stack_requirements: bool = True
|
220
202
|
apt_packages: List[str] = []
|
221
203
|
environment: Dict[str, Any] = {}
|
222
|
-
dockerignore: Optional[str] = None
|
223
|
-
copy_files: bool = True
|
224
|
-
copy_global_config: bool = True
|
225
204
|
user: Optional[str] = None
|
226
205
|
build_config: Optional[DockerBuildConfig] = None
|
227
206
|
|
228
|
-
|
207
|
+
allow_including_files_in_images: bool = True
|
208
|
+
allow_download_from_code_repository: bool = True
|
209
|
+
allow_download_from_artifact_store: bool = True
|
210
|
+
|
211
|
+
# Deprecated attributes
|
212
|
+
build_options: Dict[str, Any] = {}
|
213
|
+
dockerignore: Optional[str] = None
|
214
|
+
copy_files: bool = True
|
215
|
+
copy_global_config: bool = True
|
216
|
+
source_files: Optional[str] = None
|
217
|
+
required_hub_plugins: List[str] = []
|
229
218
|
|
230
219
|
_deprecation_validator = deprecation_utils.deprecate_pydantic_attributes(
|
231
|
-
"copy_files",
|
220
|
+
"copy_files",
|
221
|
+
"copy_global_config",
|
222
|
+
"source_files",
|
223
|
+
"required_hub_plugins",
|
232
224
|
)
|
233
225
|
|
234
226
|
@model_validator(mode="before")
|
235
227
|
@classmethod
|
236
228
|
@before_validator_handler
|
237
|
-
def
|
238
|
-
"""
|
229
|
+
def _migrate_source_files(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
230
|
+
"""Migrate old source_files values.
|
239
231
|
|
240
232
|
Args:
|
241
|
-
data: The
|
233
|
+
data: The model data.
|
234
|
+
|
235
|
+
Raises:
|
236
|
+
ValueError: If an invalid source file mode is specified.
|
242
237
|
|
243
238
|
Returns:
|
244
|
-
The migrated
|
239
|
+
The migrated data.
|
245
240
|
"""
|
246
|
-
|
241
|
+
source_files = data.get("source_files", None)
|
247
242
|
|
248
|
-
if
|
243
|
+
if source_files is None:
|
249
244
|
return data
|
250
245
|
|
251
|
-
|
252
|
-
|
246
|
+
replacement_attributes = [
|
247
|
+
"allow_including_files_in_images",
|
248
|
+
"allow_download_from_code_repository",
|
249
|
+
"allow_download_from_artifact_store",
|
250
|
+
]
|
251
|
+
if any(v in data for v in replacement_attributes):
|
253
252
|
logger.warning(
|
254
|
-
"Both `
|
255
|
-
"DockerSettings, ignoring the `
|
253
|
+
"Both `source_files` and one of %s specified for the "
|
254
|
+
"DockerSettings, ignoring the `source_files` value.",
|
255
|
+
replacement_attributes,
|
256
256
|
)
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
257
|
+
return data
|
258
|
+
|
259
|
+
allow_including_files_in_images = False
|
260
|
+
allow_download_from_code_repository = False
|
261
|
+
allow_download_from_artifact_store = False
|
262
|
+
|
263
|
+
if source_files == "download":
|
264
|
+
allow_download_from_code_repository = True
|
265
|
+
elif source_files == "include":
|
266
|
+
allow_including_files_in_images = True
|
267
|
+
elif source_files == "download_or_include":
|
268
|
+
allow_including_files_in_images = True
|
269
|
+
allow_download_from_code_repository = True
|
270
|
+
elif source_files == "ignore":
|
271
|
+
pass
|
272
|
+
else:
|
273
|
+
raise ValueError(f"Invalid source file mode `{source_files}`.")
|
274
|
+
|
275
|
+
data["allow_including_files_in_images"] = (
|
276
|
+
allow_including_files_in_images
|
277
|
+
)
|
278
|
+
data["allow_download_from_code_repository"] = (
|
279
|
+
allow_download_from_code_repository
|
280
|
+
)
|
281
|
+
data["allow_download_from_artifact_store"] = (
|
282
|
+
allow_download_from_artifact_store
|
283
|
+
)
|
261
284
|
|
262
285
|
return data
|
263
286
|
|
@@ -282,7 +305,7 @@ class DockerSettings(BaseSettings):
|
|
282
305
|
|
283
306
|
return self
|
284
307
|
|
285
|
-
model_config =
|
308
|
+
model_config = ConfigDict(
|
286
309
|
# public attributes are immutable
|
287
310
|
frozen=True,
|
288
311
|
# prevent extra attributes during model initialization
|
@@ -21,6 +21,7 @@ from pydantic import Field, SerializeAsAny
|
|
21
21
|
from zenml.config.base_settings import BaseSettings
|
22
22
|
from zenml.config.retry_config import StepRetryConfig
|
23
23
|
from zenml.config.schedule import Schedule
|
24
|
+
from zenml.config.source import SourceWithValidator
|
24
25
|
from zenml.config.step_configurations import StepConfigurationUpdate
|
25
26
|
from zenml.config.strict_base_model import StrictBaseModel
|
26
27
|
from zenml.model.model import Model
|
@@ -48,3 +49,5 @@ class PipelineRunConfiguration(
|
|
48
49
|
model: Optional[Model] = None
|
49
50
|
parameters: Optional[Dict[str, Any]] = None
|
50
51
|
retry: Optional[StepRetryConfig] = None
|
52
|
+
failure_hook_source: Optional[SourceWithValidator] = None
|
53
|
+
success_hook_source: Optional[SourceWithValidator] = None
|
zenml/config/server_config.py
CHANGED
@@ -65,6 +65,9 @@ def generate_jwt_secret_key() -> str:
|
|
65
65
|
class ServerConfiguration(BaseModel):
|
66
66
|
"""ZenML Server configuration attributes.
|
67
67
|
|
68
|
+
All these attributes can be set through the environment with the `ZENML_SERVER_`-Prefix.
|
69
|
+
The value of the `ZENML_SERVER_DEPLOYMENT_TYPE` environment variable will be extracted to deployment_type.
|
70
|
+
|
68
71
|
Attributes:
|
69
72
|
deployment_type: The type of ZenML server deployment that is running.
|
70
73
|
server_url: The URL where the ZenML server API is reachable. Must be
|
zenml/config/source.py
CHANGED
@@ -42,6 +42,7 @@ class SourceType(Enum):
|
|
42
42
|
INTERNAL = "internal"
|
43
43
|
DISTRIBUTION_PACKAGE = "distribution_package"
|
44
44
|
CODE_REPOSITORY = "code_repository"
|
45
|
+
NOTEBOOK = "notebook"
|
45
46
|
UNKNOWN = "unknown"
|
46
47
|
|
47
48
|
|
@@ -229,6 +230,63 @@ class CodeRepositorySource(Source):
|
|
229
230
|
return value
|
230
231
|
|
231
232
|
|
233
|
+
class NotebookSource(Source):
|
234
|
+
"""Source representing an object defined in a notebook.
|
235
|
+
|
236
|
+
Attributes:
|
237
|
+
code_path: Path where the notebook cell code for this source is
|
238
|
+
uploaded.
|
239
|
+
replacement_module: Name of the module from which this source should
|
240
|
+
be loaded in case the code is not running in a notebook.
|
241
|
+
"""
|
242
|
+
|
243
|
+
code_path: Optional[str] = None
|
244
|
+
replacement_module: Optional[str] = None
|
245
|
+
type: SourceType = SourceType.NOTEBOOK
|
246
|
+
|
247
|
+
# Private attribute that is used to store the code but should not be
|
248
|
+
# serialized
|
249
|
+
_cell_code: Optional[str] = None
|
250
|
+
|
251
|
+
@field_validator("type")
|
252
|
+
@classmethod
|
253
|
+
def _validate_type(cls, value: SourceType) -> SourceType:
|
254
|
+
"""Validate the source type.
|
255
|
+
|
256
|
+
Args:
|
257
|
+
value: The source type.
|
258
|
+
|
259
|
+
Raises:
|
260
|
+
ValueError: If the source type is not `NOTEBOOK`.
|
261
|
+
|
262
|
+
Returns:
|
263
|
+
The source type.
|
264
|
+
"""
|
265
|
+
if value != SourceType.NOTEBOOK:
|
266
|
+
raise ValueError("Invalid source type.")
|
267
|
+
|
268
|
+
return value
|
269
|
+
|
270
|
+
@field_validator("module")
|
271
|
+
@classmethod
|
272
|
+
def _validate_module(cls, value: str) -> str:
|
273
|
+
"""Validate the module.
|
274
|
+
|
275
|
+
Args:
|
276
|
+
value: The module.
|
277
|
+
|
278
|
+
Raises:
|
279
|
+
ValueError: If the module is not `__main__`.
|
280
|
+
|
281
|
+
Returns:
|
282
|
+
The module.
|
283
|
+
"""
|
284
|
+
if value != "__main__":
|
285
|
+
raise ValueError("Invalid module for notebook source.")
|
286
|
+
|
287
|
+
return value
|
288
|
+
|
289
|
+
|
232
290
|
def convert_source(source: Any) -> Any:
|
233
291
|
"""Converts an old source string to a source object.
|
234
292
|
|
@@ -245,5 +303,6 @@ def convert_source(source: Any) -> Any:
|
|
245
303
|
|
246
304
|
|
247
305
|
SourceWithValidator = Annotated[
|
248
|
-
SerializeAsAny[Source],
|
306
|
+
SerializeAsAny[Source],
|
307
|
+
BeforeValidator(convert_source),
|
249
308
|
]
|
zenml/constants.py
CHANGED
@@ -165,7 +165,6 @@ ENV_ZENML_SKIP_IMAGE_BUILDER_DEFAULT = "ZENML_SKIP_IMAGE_BUILDER_DEFAULT"
|
|
165
165
|
ENV_ZENML_REQUIRES_CODE_DOWNLOAD = "ZENML_REQUIRES_CODE_DOWNLOAD"
|
166
166
|
ENV_ZENML_SERVER = "ZENML_SERVER"
|
167
167
|
ENV_ZENML_LOCAL_SERVER = "ZENML_LOCAL_SERVER"
|
168
|
-
ENV_ZENML_HUB_URL = "ZENML_HUB_URL"
|
169
168
|
ENV_ZENML_ENFORCE_TYPE_ANNOTATIONS = "ZENML_ENFORCE_TYPE_ANNOTATIONS"
|
170
169
|
ENV_ZENML_ENABLE_IMPLICIT_AUTH_METHODS = "ZENML_ENABLE_IMPLICIT_AUTH_METHODS"
|
171
170
|
ENV_ZENML_DISABLE_STEP_LOGS_STORAGE = "ZENML_DISABLE_STEP_LOGS_STORAGE"
|
@@ -279,7 +278,6 @@ DEFAULT_ZENML_SERVER_SECURE_HEADERS_CONTENT = "nosniff"
|
|
279
278
|
_csp_script_src_urls = ["https://widgets-v3.featureos.app"]
|
280
279
|
_csp_connect_src_urls = [
|
281
280
|
"https://sdkdocs.zenml.io",
|
282
|
-
"https://hubapi.zenml.io",
|
283
281
|
"https://analytics.zenml.io",
|
284
282
|
]
|
285
283
|
_csp_img_src_urls = [
|
@@ -315,6 +313,7 @@ DEFAULT_ZENML_SERVER_SECURE_HEADERS_PERMISSIONS = (
|
|
315
313
|
)
|
316
314
|
DEFAULT_ZENML_SERVER_SECURE_HEADERS_REPORT_TO = "default"
|
317
315
|
DEFAULT_ZENML_SERVER_USE_LEGACY_DASHBOARD = False
|
316
|
+
DEFAULT_ZENML_SERVER_REPORT_USER_ACTIVITY_TO_DB_SECONDS = 30
|
318
317
|
|
319
318
|
# Configurations to decide which resources report their usage and check for
|
320
319
|
# entitlement in the case of a cloud deployment. Expected Format is this:
|
@@ -364,6 +363,7 @@ PIPELINES = "/pipelines"
|
|
364
363
|
PIPELINE_SPEC = "/pipeline-spec"
|
365
364
|
PLUGIN_FLAVORS = "/plugin-flavors"
|
366
365
|
RUNS = "/runs"
|
366
|
+
RUN_TEMPLATES = "/run_templates"
|
367
367
|
RUN_METADATA = "/run-metadata"
|
368
368
|
SCHEDULES = "/schedules"
|
369
369
|
SECRETS = "/secrets"
|
@@ -424,6 +424,7 @@ PAGE_SIZE_MAXIMUM: int = handle_int_env_var(
|
|
424
424
|
ENV_ZENML_PAGINATION_DEFAULT_LIMIT, default=10000
|
425
425
|
)
|
426
426
|
FILTERING_DATETIME_FORMAT: str = "%Y-%m-%d %H:%M:%S"
|
427
|
+
SORT_PIPELINES_BY_LATEST_RUN_KEY = "latest_run"
|
427
428
|
|
428
429
|
# Metadata constants
|
429
430
|
METADATA_ORCHESTRATOR_URL = "orchestrator_url"
|
@@ -492,3 +493,11 @@ BANNED_NAME_CHARACTERS = "\t\n\r\v\f"
|
|
492
493
|
|
493
494
|
|
494
495
|
STACK_DEPLOYMENT_API_TOKEN_EXPIRATION = 60 * 6 # 6 hours
|
496
|
+
|
497
|
+
# ZenML Pro
|
498
|
+
ZENML_PRO_CONNECTION_ISSUES_SUSPENDED_PAUSED_TENANT_HINT = (
|
499
|
+
"\nHINT: Since you are trying to communicate with the ZenML Pro Tenant, "
|
500
|
+
"please make sure that your tenant is in RUNNING state on your "
|
501
|
+
"Organization page. If the tenant is PAUSED you can `Resume` it via UI "
|
502
|
+
"and try again."
|
503
|
+
)
|
@@ -27,10 +27,15 @@ from zenml.constants import (
|
|
27
27
|
handle_bool_env_var,
|
28
28
|
)
|
29
29
|
from zenml.logger import get_logger
|
30
|
-
from zenml.utils import
|
30
|
+
from zenml.utils import (
|
31
|
+
code_repository_utils,
|
32
|
+
code_utils,
|
33
|
+
source_utils,
|
34
|
+
uuid_utils,
|
35
|
+
)
|
31
36
|
|
32
37
|
if TYPE_CHECKING:
|
33
|
-
from zenml.models import PipelineDeploymentResponse
|
38
|
+
from zenml.models import CodeReferenceResponse, PipelineDeploymentResponse
|
34
39
|
|
35
40
|
logger = get_logger(__name__)
|
36
41
|
DEFAULT_ENTRYPOINT_COMMAND = [
|
@@ -198,7 +203,7 @@ class BaseEntrypointConfiguration(ABC):
|
|
198
203
|
|
199
204
|
Raises:
|
200
205
|
RuntimeError: If the current environment requires code download
|
201
|
-
but the deployment does not have
|
206
|
+
but the deployment does not have a reference to any code.
|
202
207
|
"""
|
203
208
|
requires_code_download = handle_bool_env_var(
|
204
209
|
ENV_ZENML_REQUIRES_CODE_DOWNLOAD
|
@@ -207,17 +212,33 @@ class BaseEntrypointConfiguration(ABC):
|
|
207
212
|
if not requires_code_download:
|
208
213
|
return
|
209
214
|
|
210
|
-
code_reference
|
211
|
-
|
215
|
+
if code_reference := deployment.code_reference:
|
216
|
+
self.download_code_from_code_repository(
|
217
|
+
code_reference=code_reference
|
218
|
+
)
|
219
|
+
elif code_path := deployment.code_path:
|
220
|
+
self.download_code_from_artifact_store(code_path=code_path)
|
221
|
+
else:
|
212
222
|
raise RuntimeError(
|
213
|
-
"Code download required but no code reference provided."
|
223
|
+
"Code download required but no code reference or path provided."
|
214
224
|
)
|
215
225
|
|
226
|
+
logger.info("Code download finished.")
|
227
|
+
|
228
|
+
def download_code_from_code_repository(
|
229
|
+
self, code_reference: "CodeReferenceResponse"
|
230
|
+
) -> None:
|
231
|
+
"""Download code from a code repository.
|
232
|
+
|
233
|
+
Args:
|
234
|
+
code_reference: The reference to the code.
|
235
|
+
"""
|
216
236
|
logger.info(
|
217
237
|
"Downloading code from code repository `%s` (commit `%s`).",
|
218
238
|
code_reference.code_repository.name,
|
219
239
|
code_reference.commit,
|
220
240
|
)
|
241
|
+
|
221
242
|
model = Client().get_code_repository(code_reference.code_repository.id)
|
222
243
|
repo = BaseCodeRepository.from_model(model)
|
223
244
|
code_repo_root = os.path.abspath("code")
|
@@ -234,10 +255,34 @@ class BaseEntrypointConfiguration(ABC):
|
|
234
255
|
code_repository_utils.set_custom_local_repository(
|
235
256
|
root=code_repo_root, commit=code_reference.commit, repo=repo
|
236
257
|
)
|
237
|
-
|
258
|
+
|
238
259
|
sys.path.insert(0, download_dir)
|
260
|
+
os.chdir(download_dir)
|
239
261
|
|
240
|
-
|
262
|
+
def download_code_from_artifact_store(self, code_path: str) -> None:
|
263
|
+
"""Download code from the artifact store.
|
264
|
+
|
265
|
+
Args:
|
266
|
+
code_path: Path where the code is stored.
|
267
|
+
"""
|
268
|
+
logger.info(
|
269
|
+
"Downloading code from artifact store path `%s`.", code_path
|
270
|
+
)
|
271
|
+
|
272
|
+
# Do not remove this line, we need to instantiate the artifact store to
|
273
|
+
# register the filesystem needed for the file download
|
274
|
+
_ = Client().active_stack.artifact_store
|
275
|
+
|
276
|
+
extract_dir = os.path.abspath("code")
|
277
|
+
os.makedirs(extract_dir)
|
278
|
+
|
279
|
+
code_utils.download_and_extract_code(
|
280
|
+
code_path=code_path, extract_dir=extract_dir
|
281
|
+
)
|
282
|
+
|
283
|
+
source_utils.set_custom_source_root(extract_dir)
|
284
|
+
sys.path.insert(0, extract_dir)
|
285
|
+
os.chdir(extract_dir)
|
241
286
|
|
242
287
|
@abstractmethod
|
243
288
|
def run(self) -> None:
|
zenml/enums.py
CHANGED
@@ -171,7 +171,6 @@ class CliCategories(StrEnum):
|
|
171
171
|
|
172
172
|
STACK_COMPONENTS = "Stack Components"
|
173
173
|
MODEL_DEPLOYMENT = "Model Deployment"
|
174
|
-
HUB = "ZenML Hub"
|
175
174
|
INTEGRATIONS = "Integrations"
|
176
175
|
MANAGEMENT_TOOLS = "Management Tools"
|
177
176
|
MODEL_CONTROL_PLANE = "Model Control Plane"
|
@@ -303,6 +302,7 @@ class EnvironmentType(StrEnum):
|
|
303
302
|
NOTEBOOK = "notebook"
|
304
303
|
PAPERSPACE = "paperspace"
|
305
304
|
WSL = "wsl"
|
305
|
+
LIGHTNING_AI_STUDIO = "lightning_ai_studio"
|
306
306
|
|
307
307
|
|
308
308
|
class ModelStages(StrEnum):
|
@@ -338,6 +338,9 @@ class TaggableResourceTypes(StrEnum):
|
|
338
338
|
ARTIFACT_VERSION = "artifact_version"
|
339
339
|
MODEL = "model"
|
340
340
|
MODEL_VERSION = "model_version"
|
341
|
+
PIPELINE = "pipeline"
|
342
|
+
PIPELINE_RUN = "pipeline_run"
|
343
|
+
RUN_TEMPLATE = "run_template"
|
341
344
|
|
342
345
|
|
343
346
|
class ResponseUpdateStrategy(StrEnum):
|
zenml/environment.py
CHANGED
@@ -15,7 +15,6 @@
|
|
15
15
|
|
16
16
|
import os
|
17
17
|
import platform
|
18
|
-
from importlib.util import find_spec
|
19
18
|
from pathlib import Path
|
20
19
|
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, cast
|
21
20
|
|
@@ -70,6 +69,8 @@ def get_environment() -> str:
|
|
70
69
|
return EnvironmentType.BITBUCKET_CI
|
71
70
|
elif Environment.in_ci():
|
72
71
|
return EnvironmentType.GENERIC_CI
|
72
|
+
elif Environment.in_lightning_ai_studio():
|
73
|
+
return EnvironmentType.LIGHTNING_AI_STUDIO
|
73
74
|
elif Environment.in_docker():
|
74
75
|
return EnvironmentType.DOCKER
|
75
76
|
elif Environment.in_container():
|
@@ -257,15 +258,17 @@ class Environment(metaclass=SingletonMetaClass):
|
|
257
258
|
if Environment.in_google_colab():
|
258
259
|
return True
|
259
260
|
|
260
|
-
|
261
|
-
|
261
|
+
try:
|
262
|
+
ipython = get_ipython() # type: ignore[name-defined]
|
263
|
+
except NameError:
|
264
|
+
return False
|
262
265
|
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
266
|
+
if ipython.__class__.__name__ in [
|
267
|
+
"TerminalInteractiveShell",
|
268
|
+
"ZMQInteractiveShell",
|
269
|
+
"DatabricksShell",
|
270
|
+
]:
|
271
|
+
return True
|
269
272
|
return False
|
270
273
|
|
271
274
|
@staticmethod
|
@@ -339,6 +342,19 @@ class Environment(metaclass=SingletonMetaClass):
|
|
339
342
|
"""
|
340
343
|
return "microsoft-standard" in platform.uname().release
|
341
344
|
|
345
|
+
@staticmethod
|
346
|
+
def in_lightning_ai_studio() -> bool:
|
347
|
+
"""If the current Python process is running in Lightning.ai studios.
|
348
|
+
|
349
|
+
Returns:
|
350
|
+
`True` if the current Python process is running in Lightning.ai studios,
|
351
|
+
`False` otherwise.
|
352
|
+
"""
|
353
|
+
return (
|
354
|
+
"LIGHTNING_CLOUD_URL" in os.environ
|
355
|
+
and "LIGHTNING_CLOUDSPACE_HOST" in os.environ
|
356
|
+
)
|
357
|
+
|
342
358
|
def register_component(
|
343
359
|
self, component: "BaseEnvironmentComponent"
|
344
360
|
) -> "BaseEnvironmentComponent":
|
@@ -119,7 +119,7 @@ class BaseImageBuilder(StackComponent, ABC):
|
|
119
119
|
|
120
120
|
hash_ = hashlib.sha1() # nosec
|
121
121
|
with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
|
122
|
-
build_context.write_archive(f,
|
122
|
+
build_context.write_archive(f, use_gzip=True)
|
123
123
|
|
124
124
|
while True:
|
125
125
|
data = f.read(64 * 1024)
|