zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
zenml/steps/base_step.py
CHANGED
@@ -22,6 +22,7 @@ from typing import (
|
|
22
22
|
TYPE_CHECKING,
|
23
23
|
Any,
|
24
24
|
Dict,
|
25
|
+
List,
|
25
26
|
Mapping,
|
26
27
|
Optional,
|
27
28
|
Sequence,
|
@@ -38,7 +39,11 @@ from pydantic import BaseModel, ConfigDict, ValidationError
|
|
38
39
|
from zenml.client_lazy_loader import ClientLazyLoader
|
39
40
|
from zenml.config.retry_config import StepRetryConfig
|
40
41
|
from zenml.config.source import Source
|
41
|
-
from zenml.constants import
|
42
|
+
from zenml.constants import (
|
43
|
+
ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK,
|
44
|
+
STEP_SOURCE_PARAMETER_NAME,
|
45
|
+
handle_bool_env_var,
|
46
|
+
)
|
42
47
|
from zenml.exceptions import MissingStepParameterError, StepInterfaceError
|
43
48
|
from zenml.logger import get_logger
|
44
49
|
from zenml.materializers.base_materializer import BaseMaterializer
|
@@ -51,15 +56,16 @@ from zenml.steps.entrypoint_function_utils import (
|
|
51
56
|
)
|
52
57
|
from zenml.steps.utils import (
|
53
58
|
resolve_type_annotation,
|
59
|
+
run_as_single_step_pipeline,
|
54
60
|
)
|
55
61
|
from zenml.utils import (
|
56
62
|
dict_utils,
|
63
|
+
materializer_utils,
|
57
64
|
notebook_utils,
|
58
65
|
pydantic_utils,
|
59
66
|
settings_utils,
|
60
67
|
source_code_utils,
|
61
68
|
source_utils,
|
62
|
-
typing_utils,
|
63
69
|
)
|
64
70
|
|
65
71
|
if TYPE_CHECKING:
|
@@ -586,9 +592,16 @@ class BaseStep(metaclass=BaseStepMeta):
|
|
586
592
|
from zenml.new.pipelines.pipeline import Pipeline
|
587
593
|
|
588
594
|
if not Pipeline.ACTIVE_PIPELINE:
|
589
|
-
# The step is being called outside the context of a pipeline,
|
590
|
-
#
|
591
|
-
|
595
|
+
# The step is being called outside the context of a pipeline, either
|
596
|
+
# run the step function or run it as a single step pipeline on the
|
597
|
+
# active stack
|
598
|
+
run_without_stack = handle_bool_env_var(
|
599
|
+
ENV_ZENML_RUN_SINGLE_STEPS_WITHOUT_STACK, default=False
|
600
|
+
)
|
601
|
+
if run_without_stack:
|
602
|
+
return self.call_entrypoint(*args, **kwargs)
|
603
|
+
else:
|
604
|
+
return run_as_single_step_pipeline(self, *args, **kwargs)
|
592
605
|
|
593
606
|
(
|
594
607
|
input_artifacts,
|
@@ -1098,6 +1111,11 @@ To avoid this consider setting step parameters only in one place (config or code
|
|
1098
1111
|
this step.
|
1099
1112
|
client_lazy_loaders: The client lazy loaders of this step.
|
1100
1113
|
|
1114
|
+
Raises:
|
1115
|
+
StepInterfaceError: If explicit materializers were specified for an
|
1116
|
+
output but they do not work for the data type(s) defined by
|
1117
|
+
the type annotation.
|
1118
|
+
|
1101
1119
|
Returns:
|
1102
1120
|
The finalized step configuration.
|
1103
1121
|
"""
|
@@ -1119,9 +1137,45 @@ To avoid this consider setting step parameters only in one place (config or code
|
|
1119
1137
|
output_name, PartialArtifactConfiguration()
|
1120
1138
|
)
|
1121
1139
|
|
1122
|
-
|
1140
|
+
if output.materializer_source:
|
1141
|
+
# The materializer source was configured by the user. We
|
1142
|
+
# validate that their configured materializer supports the
|
1143
|
+
# output type. If the output annotation is a Union, we check
|
1144
|
+
# that at least one of the specified materializers works with at
|
1145
|
+
# least one of the types in the Union. If that's not the case,
|
1146
|
+
# it would be a guaranteed failure at runtime and we fail early
|
1147
|
+
# here.
|
1148
|
+
if output_annotation.resolved_annotation is Any:
|
1149
|
+
continue
|
1123
1150
|
|
1124
|
-
|
1151
|
+
materializer_classes: List[Type["BaseMaterializer"]] = [
|
1152
|
+
source_utils.load(materializer_source)
|
1153
|
+
for materializer_source in output.materializer_source
|
1154
|
+
]
|
1155
|
+
|
1156
|
+
for data_type in output_annotation.get_output_types():
|
1157
|
+
try:
|
1158
|
+
materializer_utils.select_materializer(
|
1159
|
+
data_type=data_type,
|
1160
|
+
materializer_classes=materializer_classes,
|
1161
|
+
)
|
1162
|
+
break
|
1163
|
+
except RuntimeError:
|
1164
|
+
pass
|
1165
|
+
else:
|
1166
|
+
materializer_strings = [
|
1167
|
+
materializer_source.import_path
|
1168
|
+
for materializer_source in output.materializer_source
|
1169
|
+
]
|
1170
|
+
raise StepInterfaceError(
|
1171
|
+
"Invalid materializers specified for output "
|
1172
|
+
f"{output_name} of step {self.name}. None of the "
|
1173
|
+
f"materializers ({materializer_strings}) are "
|
1174
|
+
"able to save or load data of the type that is defined "
|
1175
|
+
"for the output "
|
1176
|
+
f"({output_annotation.resolved_annotation})."
|
1177
|
+
)
|
1178
|
+
else:
|
1125
1179
|
if output_annotation.resolved_annotation is Any:
|
1126
1180
|
outputs[output_name]["materializer_source"] = ()
|
1127
1181
|
outputs[output_name]["default_materializer_source"] = (
|
@@ -1131,26 +1185,9 @@ To avoid this consider setting step parameters only in one place (config or code
|
|
1131
1185
|
)
|
1132
1186
|
continue
|
1133
1187
|
|
1134
|
-
if typing_utils.is_union(
|
1135
|
-
typing_utils.get_origin(
|
1136
|
-
output_annotation.resolved_annotation
|
1137
|
-
)
|
1138
|
-
or output_annotation.resolved_annotation
|
1139
|
-
):
|
1140
|
-
output_types = tuple(
|
1141
|
-
type(None)
|
1142
|
-
if typing_utils.is_none_type(output_type)
|
1143
|
-
else output_type
|
1144
|
-
for output_type in get_args(
|
1145
|
-
output_annotation.resolved_annotation
|
1146
|
-
)
|
1147
|
-
)
|
1148
|
-
else:
|
1149
|
-
output_types = (output_annotation.resolved_annotation,)
|
1150
|
-
|
1151
1188
|
materializer_sources = []
|
1152
1189
|
|
1153
|
-
for output_type in
|
1190
|
+
for output_type in output_annotation.get_output_types():
|
1154
1191
|
materializer_class = materializer_registry[output_type]
|
1155
1192
|
materializer_sources.append(
|
1156
1193
|
source_utils.resolve(materializer_class)
|
zenml/steps/utils.py
CHANGED
@@ -18,7 +18,7 @@ import ast
|
|
18
18
|
import contextlib
|
19
19
|
import inspect
|
20
20
|
import textwrap
|
21
|
-
from typing import Any, Callable, Dict, Optional, Tuple, Union
|
21
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union
|
22
22
|
from uuid import UUID
|
23
23
|
|
24
24
|
from pydantic import BaseModel
|
@@ -26,12 +26,17 @@ from typing_extensions import Annotated
|
|
26
26
|
|
27
27
|
from zenml.artifacts.artifact_config import ArtifactConfig
|
28
28
|
from zenml.client import Client
|
29
|
-
from zenml.enums import MetadataResourceTypes
|
29
|
+
from zenml.enums import ExecutionStatus, MetadataResourceTypes
|
30
|
+
from zenml.exceptions import StepInterfaceError
|
30
31
|
from zenml.logger import get_logger
|
31
32
|
from zenml.metadata.metadata_types import MetadataType
|
32
33
|
from zenml.new.steps.step_context import get_step_context
|
33
34
|
from zenml.steps.step_output import Output
|
34
|
-
from zenml.utils import source_code_utils, typing_utils
|
35
|
+
from zenml.utils import settings_utils, source_code_utils, typing_utils
|
36
|
+
|
37
|
+
if TYPE_CHECKING:
|
38
|
+
from zenml.steps import BaseStep
|
39
|
+
|
35
40
|
|
36
41
|
logger = get_logger(__name__)
|
37
42
|
|
@@ -45,6 +50,28 @@ class OutputSignature(BaseModel):
|
|
45
50
|
artifact_config: Optional[ArtifactConfig] = None
|
46
51
|
has_custom_name: bool = False
|
47
52
|
|
53
|
+
def get_output_types(self) -> Tuple[Any, ...]:
|
54
|
+
"""Get all output types that match the type annotation.
|
55
|
+
|
56
|
+
Returns:
|
57
|
+
All output types that match the type annotation.
|
58
|
+
"""
|
59
|
+
if self.resolved_annotation is Any:
|
60
|
+
return ()
|
61
|
+
|
62
|
+
if typing_utils.is_union(
|
63
|
+
typing_utils.get_origin(self.resolved_annotation)
|
64
|
+
or self.resolved_annotation
|
65
|
+
):
|
66
|
+
return tuple(
|
67
|
+
type(None)
|
68
|
+
if typing_utils.is_none_type(output_type)
|
69
|
+
else output_type
|
70
|
+
for output_type in get_args(self.resolved_annotation)
|
71
|
+
)
|
72
|
+
else:
|
73
|
+
return (self.resolved_annotation,)
|
74
|
+
|
48
75
|
|
49
76
|
def get_args(obj: Any) -> Tuple[Any, ...]:
|
50
77
|
"""Get arguments of a type annotation.
|
@@ -464,3 +491,88 @@ def log_step_metadata(
|
|
464
491
|
resource_id=step_run_id,
|
465
492
|
resource_type=MetadataResourceTypes.STEP_RUN,
|
466
493
|
)
|
494
|
+
|
495
|
+
|
496
|
+
def run_as_single_step_pipeline(
|
497
|
+
__step: "BaseStep", *args: Any, **kwargs: Any
|
498
|
+
) -> Any:
|
499
|
+
"""Runs the step as a single step pipeline.
|
500
|
+
|
501
|
+
- All inputs that are not JSON serializable will be uploaded to the
|
502
|
+
artifact store before the pipeline is being executed.
|
503
|
+
- All output artifacts of the step will be loaded using the materializer
|
504
|
+
that was used to store them.
|
505
|
+
|
506
|
+
Args:
|
507
|
+
*args: Entrypoint function arguments.
|
508
|
+
**kwargs: Entrypoint function keyword arguments.
|
509
|
+
|
510
|
+
Raises:
|
511
|
+
RuntimeError: If the step execution failed.
|
512
|
+
StepInterfaceError: If the arguments to the entrypoint function are
|
513
|
+
invalid.
|
514
|
+
|
515
|
+
Returns:
|
516
|
+
The output of the step entrypoint function.
|
517
|
+
"""
|
518
|
+
from zenml import ExternalArtifact, pipeline
|
519
|
+
from zenml.config.base_settings import BaseSettings
|
520
|
+
from zenml.new.pipelines.run_utils import (
|
521
|
+
wait_for_pipeline_run_to_finish,
|
522
|
+
)
|
523
|
+
|
524
|
+
logger.info(
|
525
|
+
"Running single step pipeline to execute step `%s`", __step.name
|
526
|
+
)
|
527
|
+
|
528
|
+
try:
|
529
|
+
validated_arguments = (
|
530
|
+
inspect.signature(__step.entrypoint)
|
531
|
+
.bind(*args, **kwargs)
|
532
|
+
.arguments
|
533
|
+
)
|
534
|
+
except TypeError as e:
|
535
|
+
raise StepInterfaceError(
|
536
|
+
"Invalid step function entrypoint arguments. Check out the "
|
537
|
+
"error above for more details."
|
538
|
+
) from e
|
539
|
+
|
540
|
+
inputs: Dict[str, Any] = {}
|
541
|
+
for key, value in validated_arguments.items():
|
542
|
+
try:
|
543
|
+
__step.entrypoint_definition.validate_input(key=key, value=value)
|
544
|
+
inputs[key] = value
|
545
|
+
except Exception:
|
546
|
+
inputs[key] = ExternalArtifact(value=value)
|
547
|
+
|
548
|
+
orchestrator = Client().active_stack.orchestrator
|
549
|
+
|
550
|
+
pipeline_settings: Any = {}
|
551
|
+
if "synchronous" in orchestrator.config.model_fields:
|
552
|
+
# Make sure the orchestrator runs sync so we stream the logs
|
553
|
+
key = settings_utils.get_stack_component_setting_key(orchestrator)
|
554
|
+
pipeline_settings[key] = BaseSettings(synchronous=True)
|
555
|
+
|
556
|
+
@pipeline(name=__step.name, enable_cache=False, settings=pipeline_settings)
|
557
|
+
def single_step_pipeline() -> None:
|
558
|
+
__step(**inputs)
|
559
|
+
|
560
|
+
run = single_step_pipeline.with_options(unlisted=True)()
|
561
|
+
run = wait_for_pipeline_run_to_finish(run.id)
|
562
|
+
|
563
|
+
if run.status != ExecutionStatus.COMPLETED:
|
564
|
+
raise RuntimeError("Failed to execute step %s.", __step.name)
|
565
|
+
|
566
|
+
# 4. Load output artifacts
|
567
|
+
step_run = next(iter(run.steps.values()))
|
568
|
+
outputs = [
|
569
|
+
step_run.outputs[output_name].load()
|
570
|
+
for output_name in step_run.config.outputs.keys()
|
571
|
+
]
|
572
|
+
|
573
|
+
if len(outputs) == 0:
|
574
|
+
return None
|
575
|
+
elif len(outputs) == 1:
|
576
|
+
return outputs[0]
|
577
|
+
else:
|
578
|
+
return tuple(outputs)
|
zenml/utils/cloud_utils.py
CHANGED
@@ -20,21 +20,21 @@ from zenml.utils.dashboard_utils import get_model_version_url
|
|
20
20
|
logger = get_logger(__name__)
|
21
21
|
|
22
22
|
|
23
|
-
def
|
24
|
-
"""Check if a model version is from a ZenML Pro server.
|
23
|
+
def try_get_model_version_url(model_version: ModelVersionResponse) -> str:
|
24
|
+
"""Check if a model version is from a ZenML Pro server and return its' URL.
|
25
25
|
|
26
26
|
Args:
|
27
27
|
model_version: The model version to check.
|
28
28
|
|
29
29
|
Returns:
|
30
|
-
|
30
|
+
URL if the model version is from a ZenML Pro server, else empty string.
|
31
31
|
"""
|
32
32
|
model_version_url = get_model_version_url(model_version.id)
|
33
33
|
if model_version_url:
|
34
|
-
|
35
|
-
|
36
|
-
f"
|
34
|
+
return (
|
35
|
+
"Dashboard URL for created Model Version "
|
36
|
+
f"`{model_version.model.name}::{model_version.name}`:\n"
|
37
|
+
+ model_version_url
|
37
38
|
)
|
38
|
-
return True
|
39
39
|
else:
|
40
|
-
return
|
40
|
+
return ""
|
zenml/utils/code_utils.py
CHANGED
@@ -16,6 +16,7 @@
|
|
16
16
|
import hashlib
|
17
17
|
import os
|
18
18
|
import shutil
|
19
|
+
import sys
|
19
20
|
import tempfile
|
20
21
|
from pathlib import Path
|
21
22
|
from typing import IO, TYPE_CHECKING, Dict, Optional
|
@@ -23,12 +24,14 @@ from typing import IO, TYPE_CHECKING, Dict, Optional
|
|
23
24
|
from zenml.client import Client
|
24
25
|
from zenml.io import fileio
|
25
26
|
from zenml.logger import get_logger
|
26
|
-
from zenml.utils import string_utils
|
27
|
+
from zenml.utils import source_utils, string_utils
|
27
28
|
from zenml.utils.archivable import Archivable
|
28
29
|
|
29
30
|
if TYPE_CHECKING:
|
30
31
|
from git.repo.base import Repo
|
31
32
|
|
33
|
+
from zenml.artifact_stores import BaseArtifactStore
|
34
|
+
|
32
35
|
|
33
36
|
logger = get_logger(__name__)
|
34
37
|
|
@@ -168,6 +171,30 @@ class CodeArchive(Archivable):
|
|
168
171
|
)
|
169
172
|
|
170
173
|
|
174
|
+
def compute_file_hash(file: IO[bytes]) -> str:
|
175
|
+
"""Compute a hash of the content of a file.
|
176
|
+
|
177
|
+
This function will not seek the file before or after the hash computation.
|
178
|
+
This means that the content will be computed based on the current cursor
|
179
|
+
until the end of the file.
|
180
|
+
|
181
|
+
Args:
|
182
|
+
file: The file for which to compute the hash.
|
183
|
+
|
184
|
+
Returns:
|
185
|
+
A hash of the file content.
|
186
|
+
"""
|
187
|
+
hash_ = hashlib.sha1() # nosec
|
188
|
+
|
189
|
+
while True:
|
190
|
+
data = file.read(64 * 1024)
|
191
|
+
if not data:
|
192
|
+
break
|
193
|
+
hash_.update(data)
|
194
|
+
|
195
|
+
return hash_.hexdigest()
|
196
|
+
|
197
|
+
|
171
198
|
def upload_code_if_necessary(code_archive: CodeArchive) -> str:
|
172
199
|
"""Upload code to the artifact store if necessary.
|
173
200
|
|
@@ -179,7 +206,7 @@ def upload_code_if_necessary(code_archive: CodeArchive) -> str:
|
|
179
206
|
code_archive: The code archive to upload.
|
180
207
|
|
181
208
|
Returns:
|
182
|
-
The path where
|
209
|
+
The path where the archived code is uploaded.
|
183
210
|
"""
|
184
211
|
artifact_store = Client().active_stack.artifact_store
|
185
212
|
|
@@ -187,36 +214,27 @@ def upload_code_if_necessary(code_archive: CodeArchive) -> str:
|
|
187
214
|
mode="w+b", delete=False, suffix=".tar.gz"
|
188
215
|
) as f:
|
189
216
|
code_archive.write_archive(f)
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
fileio.copy(f.name, upload_path)
|
212
|
-
logger.info("Code upload finished.")
|
213
|
-
else:
|
214
|
-
logger.info(
|
215
|
-
"Code already exists in artifact store, skipping upload."
|
216
|
-
)
|
217
|
-
|
218
|
-
if os.path.exists(f.name):
|
219
|
-
os.remove(f.name)
|
217
|
+
archive_path = f.name
|
218
|
+
archive_hash = compute_file_hash(f)
|
219
|
+
|
220
|
+
upload_dir = os.path.join(artifact_store.path, "code_uploads")
|
221
|
+
fileio.makedirs(upload_dir)
|
222
|
+
upload_path = os.path.join(upload_dir, f"{archive_hash}.tar.gz")
|
223
|
+
|
224
|
+
if not fileio.exists(upload_path):
|
225
|
+
archive_size = string_utils.get_human_readable_filesize(
|
226
|
+
os.path.getsize(archive_path)
|
227
|
+
)
|
228
|
+
logger.info(
|
229
|
+
"Uploading code to `%s` (Size: %s).", upload_path, archive_size
|
230
|
+
)
|
231
|
+
fileio.copy(archive_path, upload_path)
|
232
|
+
logger.info("Code upload finished.")
|
233
|
+
else:
|
234
|
+
logger.info("Code already exists in artifact store, skipping upload.")
|
235
|
+
|
236
|
+
if os.path.exists(archive_path):
|
237
|
+
os.remove(archive_path)
|
220
238
|
|
221
239
|
return upload_path
|
222
240
|
|
@@ -242,3 +260,83 @@ def download_and_extract_code(code_path: str, extract_dir: str) -> None:
|
|
242
260
|
|
243
261
|
shutil.unpack_archive(filename=download_path, extract_dir=extract_dir)
|
244
262
|
os.remove(download_path)
|
263
|
+
|
264
|
+
|
265
|
+
def download_code_from_artifact_store(code_path: str) -> None:
|
266
|
+
"""Download code from the artifact store.
|
267
|
+
|
268
|
+
Args:
|
269
|
+
code_path: Path where the code is stored.
|
270
|
+
"""
|
271
|
+
logger.info("Downloading code from artifact store path `%s`.", code_path)
|
272
|
+
|
273
|
+
# Do not remove this line, we need to instantiate the artifact store to
|
274
|
+
# register the filesystem needed for the file download
|
275
|
+
_ = Client().active_stack.artifact_store
|
276
|
+
|
277
|
+
extract_dir = os.path.abspath("code")
|
278
|
+
os.makedirs(extract_dir)
|
279
|
+
|
280
|
+
download_and_extract_code(code_path=code_path, extract_dir=extract_dir)
|
281
|
+
|
282
|
+
source_utils.set_custom_source_root(extract_dir)
|
283
|
+
sys.path.insert(0, extract_dir)
|
284
|
+
os.chdir(extract_dir)
|
285
|
+
|
286
|
+
|
287
|
+
def _get_notebook_upload_dir(artifact_store: "BaseArtifactStore") -> str:
|
288
|
+
"""Get the upload directory for code extracted from notebook cells.
|
289
|
+
|
290
|
+
Args:
|
291
|
+
artifact_store: The artifact store in which the directory should be.
|
292
|
+
|
293
|
+
Returns:
|
294
|
+
The upload directory for code extracted from notebook cells.
|
295
|
+
"""
|
296
|
+
return os.path.join(artifact_store.path, "notebook_code")
|
297
|
+
|
298
|
+
|
299
|
+
def upload_notebook_code(
|
300
|
+
artifact_store: "BaseArtifactStore", cell_code: str, file_name: str
|
301
|
+
) -> None:
|
302
|
+
"""Upload code extracted from a notebook cell.
|
303
|
+
|
304
|
+
Args:
|
305
|
+
artifact_store: The artifact store in which to upload the code.
|
306
|
+
cell_code: The notebook cell code.
|
307
|
+
file_name: The filename to use for storing the cell code.
|
308
|
+
"""
|
309
|
+
upload_dir = _get_notebook_upload_dir(artifact_store=artifact_store)
|
310
|
+
fileio.makedirs(upload_dir)
|
311
|
+
upload_path = os.path.join(upload_dir, file_name)
|
312
|
+
|
313
|
+
if not fileio.exists(upload_path):
|
314
|
+
with fileio.open(upload_path, "wb") as f:
|
315
|
+
f.write(cell_code.encode())
|
316
|
+
|
317
|
+
logger.info("Uploaded notebook cell code to %s.", upload_path)
|
318
|
+
|
319
|
+
|
320
|
+
def download_notebook_code(
|
321
|
+
artifact_store: "BaseArtifactStore", file_name: str, download_path: str
|
322
|
+
) -> None:
|
323
|
+
"""Download code extracted from a notebook cell.
|
324
|
+
|
325
|
+
Args:
|
326
|
+
artifact_store: The artifact store from which to download the code.
|
327
|
+
file_name: The name of the code file.
|
328
|
+
download_path: The local path where the file should be downloaded to.
|
329
|
+
|
330
|
+
Raises:
|
331
|
+
FileNotFoundError: If no file with the given filename exists in this
|
332
|
+
artifact store.
|
333
|
+
"""
|
334
|
+
code_dir = _get_notebook_upload_dir(artifact_store=artifact_store)
|
335
|
+
code_path = os.path.join(code_dir, file_name)
|
336
|
+
|
337
|
+
if not fileio.exists(code_path):
|
338
|
+
raise FileNotFoundError(
|
339
|
+
f"Notebook code at path {code_path} not found."
|
340
|
+
)
|
341
|
+
|
342
|
+
fileio.copy(code_path, download_path)
|
zenml/utils/function_utils.py
CHANGED
@@ -34,12 +34,12 @@ from zenml.utils.function_utils import _cli_wrapped_function
|
|
34
34
|
import sys
|
35
35
|
sys.path.append(r"{func_path}")
|
36
36
|
|
37
|
-
from {func_module} import {func_name} as
|
37
|
+
from {func_module} import {func_name} as step_function
|
38
38
|
|
39
|
-
if
|
40
|
-
func = _cli_wrapped_function(
|
39
|
+
if unwrapped_entrypoint:=getattr(step_function, "unwrapped_entrypoint", None):
|
40
|
+
func = _cli_wrapped_function(unwrapped_entrypoint)
|
41
41
|
else:
|
42
|
-
func = _cli_wrapped_function(
|
42
|
+
func = _cli_wrapped_function(step_function.entrypoint)
|
43
43
|
"""
|
44
44
|
_CLI_WRAPPED_MAINS = {
|
45
45
|
"accelerate": """
|
@@ -204,13 +204,13 @@ def _cli_wrapped_function(func: F) -> F:
|
|
204
204
|
|
205
205
|
@contextmanager
|
206
206
|
def create_cli_wrapped_script(
|
207
|
-
func: F,
|
207
|
+
func: F, flavor: str = "accelerate"
|
208
208
|
) -> Iterator[Tuple[Path, Path]]:
|
209
209
|
"""Create a script with the CLI-wrapped function.
|
210
210
|
|
211
211
|
Args:
|
212
212
|
func: The function to use.
|
213
|
-
|
213
|
+
flavor: The flavor to use.
|
214
214
|
|
215
215
|
Yields:
|
216
216
|
The paths of the script and the output.
|
@@ -241,7 +241,7 @@ def create_cli_wrapped_script(
|
|
241
241
|
func_module=clean_module_name,
|
242
242
|
func_name=func.__name__,
|
243
243
|
)
|
244
|
-
script += _CLI_WRAPPED_MAINS[
|
244
|
+
script += _CLI_WRAPPED_MAINS[flavor].format(
|
245
245
|
output_file=str(output_path.absolute())
|
246
246
|
)
|
247
247
|
f.write(script)
|
zenml/utils/notebook_utils.py
CHANGED
@@ -13,6 +13,7 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Notebook utilities."""
|
15
15
|
|
16
|
+
import hashlib
|
16
17
|
from typing import Any, Callable, Optional, TypeVar, Union
|
17
18
|
|
18
19
|
from zenml.environment import Environment
|
@@ -120,3 +121,16 @@ def warn_about_notebook_cell_magic_commands(cell_code: str) -> None:
|
|
120
121
|
"of these lines contain Jupyter notebook magic commands, "
|
121
122
|
"remove them and try again."
|
122
123
|
)
|
124
|
+
|
125
|
+
|
126
|
+
def compute_cell_replacement_module_name(cell_code: str) -> str:
|
127
|
+
"""Compute the replacement module name for a given cell code.
|
128
|
+
|
129
|
+
Args:
|
130
|
+
cell_code: The code of the notebook cell.
|
131
|
+
|
132
|
+
Returns:
|
133
|
+
The replacement module name.
|
134
|
+
"""
|
135
|
+
code_hash = hashlib.sha1(cell_code.encode()).hexdigest() # nosec
|
136
|
+
return f"extracted_notebook_code_{code_hash}"
|
@@ -38,7 +38,6 @@ from zenml.constants import (
|
|
38
38
|
ENV_ZENML_CONFIG_PATH,
|
39
39
|
ENV_ZENML_ENABLE_REPO_INIT_WARNINGS,
|
40
40
|
ENV_ZENML_LOGGING_COLORS_DISABLED,
|
41
|
-
ENV_ZENML_REQUIRES_CODE_DOWNLOAD,
|
42
41
|
handle_bool_env_var,
|
43
42
|
)
|
44
43
|
from zenml.enums import OperatingSystemType
|
@@ -55,10 +54,7 @@ if TYPE_CHECKING:
|
|
55
54
|
logger = get_logger(__name__)
|
56
55
|
|
57
56
|
DOCKER_IMAGE_WORKDIR = "/app"
|
58
|
-
|
59
|
-
DOCKER_IMAGE_ZENML_CONFIG_PATH = (
|
60
|
-
f"{DOCKER_IMAGE_WORKDIR}/{DOCKER_IMAGE_ZENML_CONFIG_DIR}"
|
61
|
-
)
|
57
|
+
DOCKER_IMAGE_ZENML_CONFIG_PATH = f"{DOCKER_IMAGE_WORKDIR}/.zenconfig"
|
62
58
|
|
63
59
|
DEFAULT_DOCKER_PARENT_IMAGE = (
|
64
60
|
f"zenmldocker/zenml:{zenml.__version__}-"
|
@@ -334,7 +330,6 @@ class PipelineDockerImageBuilder:
|
|
334
330
|
dockerfile = self._generate_zenml_pipeline_dockerfile(
|
335
331
|
parent_image=parent_image,
|
336
332
|
docker_settings=docker_settings,
|
337
|
-
download_files=download_files,
|
338
333
|
requirements_files=requirements_files,
|
339
334
|
apt_packages=apt_packages,
|
340
335
|
entrypoint=entrypoint,
|
@@ -563,7 +558,6 @@ class PipelineDockerImageBuilder:
|
|
563
558
|
def _generate_zenml_pipeline_dockerfile(
|
564
559
|
parent_image: str,
|
565
560
|
docker_settings: DockerSettings,
|
566
|
-
download_files: bool,
|
567
561
|
requirements_files: Sequence[Tuple[str, str, List[str]]] = (),
|
568
562
|
apt_packages: Sequence[str] = (),
|
569
563
|
entrypoint: Optional[str] = None,
|
@@ -573,7 +567,6 @@ class PipelineDockerImageBuilder:
|
|
573
567
|
Args:
|
574
568
|
parent_image: The image to use as parent for the Dockerfile.
|
575
569
|
docker_settings: Docker settings for this image build.
|
576
|
-
download_files: Whether to download files in the build context.
|
577
570
|
requirements_files: List of tuples that contain three items:
|
578
571
|
- the name of a requirements file,
|
579
572
|
- the content of that file,
|
@@ -641,9 +634,6 @@ class PipelineDockerImageBuilder:
|
|
641
634
|
)
|
642
635
|
|
643
636
|
lines.append(f"ENV {ENV_ZENML_ENABLE_REPO_INIT_WARNINGS}=False")
|
644
|
-
if download_files:
|
645
|
-
lines.append(f"ENV {ENV_ZENML_REQUIRES_CODE_DOWNLOAD}=True")
|
646
|
-
|
647
637
|
lines.append(
|
648
638
|
f"ENV {ENV_ZENML_CONFIG_PATH}={DOCKER_IMAGE_ZENML_CONFIG_PATH}"
|
649
639
|
)
|
zenml/utils/pydantic_utils.py
CHANGED
@@ -297,7 +297,7 @@ def model_validator_data_handler(
|
|
297
297
|
this can create conflicts after the migration and this function will be
|
298
298
|
used as a helper function to handle different types of raw input data.
|
299
299
|
|
300
|
-
A code snippet to showcase how the
|
300
|
+
A code snippet to showcase how the behavior changes. The "before" validator
|
301
301
|
prints the type of the input:
|
302
302
|
|
303
303
|
class Base(BaseModel):
|
@@ -336,7 +336,7 @@ def model_validator_data_handler(
|
|
336
336
|
# raw data such as MyClass(...) or MyClass.model_validate()
|
337
337
|
|
338
338
|
if isinstance(raw_data, dict):
|
339
|
-
# In most cases, this is the
|
339
|
+
# In most cases, this is the behavior as the raw input is a dict
|
340
340
|
return raw_data
|
341
341
|
|
342
342
|
elif isinstance(raw_data, base_class):
|
@@ -354,7 +354,7 @@ def model_validator_data_handler(
|
|
354
354
|
f"During the validation of a `{base_class}` object, an instance"
|
355
355
|
f"of `{raw_data.__class__}` (super class of `{base_class}`) "
|
356
356
|
f"has been passed as raw input. This might lead to unexpected "
|
357
|
-
f"
|
357
|
+
f"behavior in case `{base_class}` have features which can not"
|
358
358
|
f"be extracted from an instance of a `{raw_data.__class__}`."
|
359
359
|
)
|
360
360
|
return dict(raw_data)
|