zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +1 -1
- RELEASE_NOTES.md +126 -4
- zenml/VERSION +1 -1
- zenml/artifacts/utils.py +13 -6
- zenml/cli/__init__.py +1 -1
- zenml/cli/base.py +4 -4
- zenml/cli/integration.py +48 -9
- zenml/cli/pipeline.py +9 -2
- zenml/cli/stack.py +39 -27
- zenml/cli/utils.py +13 -0
- zenml/client.py +15 -17
- zenml/config/compiler.py +34 -0
- zenml/config/server_config.py +30 -0
- zenml/config/source.py +3 -7
- zenml/constants.py +5 -3
- zenml/entrypoints/base_entrypoint_configuration.py +41 -27
- zenml/entrypoints/step_entrypoint_configuration.py +5 -2
- zenml/enums.py +2 -0
- zenml/environment.py +31 -0
- zenml/feature_stores/base_feature_store.py +4 -6
- zenml/integrations/__init__.py +3 -0
- zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
- zenml/integrations/aws/__init__.py +2 -2
- zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
- zenml/integrations/azure/__init__.py +2 -2
- zenml/integrations/azure/azureml_utils.py +201 -0
- zenml/integrations/azure/flavors/azureml.py +139 -0
- zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
- zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
- zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
- zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
- zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
- zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
- zenml/integrations/constants.py +3 -0
- zenml/integrations/databricks/__init__.py +22 -4
- zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
- zenml/integrations/deepchecks/__init__.py +29 -11
- zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
- zenml/integrations/deepchecks/validation_checks.py +0 -30
- zenml/integrations/evidently/__init__.py +17 -2
- zenml/integrations/facets/__init__.py +21 -5
- zenml/integrations/feast/__init__.py +19 -6
- zenml/integrations/gcp/__init__.py +2 -2
- zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
- zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
- zenml/integrations/great_expectations/__init__.py +21 -7
- zenml/integrations/huggingface/__init__.py +39 -15
- zenml/integrations/huggingface/materializers/__init__.py +3 -0
- zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
- zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
- zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
- zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
- zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
- zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
- zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
- zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
- zenml/integrations/lightning/__init__.py +48 -0
- zenml/integrations/lightning/flavors/__init__.py +23 -0
- zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
- zenml/integrations/lightning/orchestrators/__init__.py +23 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
- zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
- zenml/integrations/lightning/orchestrators/utils.py +67 -0
- zenml/integrations/mlflow/__init__.py +43 -5
- zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
- zenml/integrations/numpy/__init__.py +32 -0
- zenml/integrations/numpy/materializers/__init__.py +18 -0
- zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
- zenml/integrations/pandas/__init__.py +32 -0
- zenml/integrations/pandas/materializers/__init__.py +18 -0
- zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
- zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
- zenml/integrations/seldon/__init__.py +18 -3
- zenml/integrations/sklearn/__init__.py +1 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/tensorboard/__init__.py +1 -1
- zenml/integrations/tensorflow/__init__.py +2 -2
- zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
- zenml/integrations/whylogs/__init__.py +18 -2
- zenml/logging/step_logging.py +9 -2
- zenml/materializers/__init__.py +0 -4
- zenml/materializers/base_materializer.py +4 -0
- zenml/materializers/numpy_materializer.py +23 -234
- zenml/materializers/pandas_materializer.py +22 -179
- zenml/model/model.py +91 -2
- zenml/model/utils.py +5 -5
- zenml/models/__init__.py +16 -3
- zenml/models/v2/core/model_version.py +1 -1
- zenml/models/v2/core/pipeline_run.py +31 -1
- zenml/models/v2/core/stack.py +51 -20
- zenml/models/v2/core/step_run.py +28 -0
- zenml/models/v2/misc/info_models.py +78 -0
- zenml/new/pipelines/pipeline.py +65 -25
- zenml/new/pipelines/run_utils.py +57 -136
- zenml/new/steps/step_context.py +17 -6
- zenml/orchestrators/base_orchestrator.py +9 -0
- zenml/orchestrators/step_launcher.py +37 -14
- zenml/orchestrators/step_runner.py +14 -13
- zenml/orchestrators/utils.py +107 -7
- zenml/service_connectors/service_connector_utils.py +2 -2
- zenml/stack/utils.py +11 -2
- zenml/stack_deployments/azure_stack_deployment.py +2 -1
- zenml/steps/base_step.py +62 -25
- zenml/steps/utils.py +115 -3
- zenml/utils/cloud_utils.py +8 -8
- zenml/utils/code_utils.py +130 -32
- zenml/utils/function_utils.py +7 -7
- zenml/utils/notebook_utils.py +14 -0
- zenml/utils/pipeline_docker_image_builder.py +1 -11
- zenml/utils/pydantic_utils.py +3 -3
- zenml/utils/secret_utils.py +2 -2
- zenml/utils/settings_utils.py +1 -1
- zenml/utils/source_utils.py +67 -21
- zenml/utils/string_utils.py +29 -0
- zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
- zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
- zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
- zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
- zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
- zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
- zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
- zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
- zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
- zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
- zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
- zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
- zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
- zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
- zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
- zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
- zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
- zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
- zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
- zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
- zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
- zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
- zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
- zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
- zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
- zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
- zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
- zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
- zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
- zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
- zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
- zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
- zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
- zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
- zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
- zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
- zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
- zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
- zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
- zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
- zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
- zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
- zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
- zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
- zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
- zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
- zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
- zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
- zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
- zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
- zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
- zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
- zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
- zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
- zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
- zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
- zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
- zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
- zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
- zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
- zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
- zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
- zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
- zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
- zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
- zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
- zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
- zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
- zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
- zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
- zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
- zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
- zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
- zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
- zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
- zenml/zen_server/routers/workspaces_endpoints.py +20 -66
- zenml/zen_server/secure_headers.py +120 -0
- zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
- zenml/zen_server/template_execution/utils.py +1 -0
- zenml/zen_server/utils.py +0 -100
- zenml/zen_server/zen_server_api.py +4 -2
- zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
- zenml/zen_stores/rest_zen_store.py +4 -21
- zenml/zen_stores/schemas/constants.py +16 -0
- zenml/zen_stores/schemas/model_schemas.py +9 -3
- zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
- zenml/zen_stores/schemas/step_run_schemas.py +23 -12
- zenml/zen_stores/sql_zen_store.py +312 -300
- zenml/zen_stores/zen_store_interface.py +0 -16
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
- zenml/models/v2/misc/full_stack.py +0 -129
- zenml/new/pipelines/model_utils.py +0 -72
- zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
- zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
- zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
- zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
- zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
- zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
- zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
- zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
- zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
- zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
- zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
- zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
- zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
- zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
- zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
- zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
- zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
- zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
- zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
- zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
- zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
- zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
- zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
- zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
- zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
- zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
- zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
- zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
- zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
- zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
- zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
- zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
- zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
- zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -13,11 +13,10 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Initialization of the Databricks integration for ZenML."""
|
15
15
|
|
16
|
-
from typing import List, Type
|
16
|
+
from typing import List, Type, Optional
|
17
|
+
|
18
|
+
from zenml.integrations.constants import DATABRICKS
|
17
19
|
|
18
|
-
from zenml.integrations.constants import (
|
19
|
-
DATABRICKS,
|
20
|
-
)
|
21
20
|
from zenml.integrations.integration import Integration
|
22
21
|
from zenml.stack import Flavor
|
23
22
|
|
@@ -32,6 +31,25 @@ class DatabricksIntegration(Integration):
|
|
32
31
|
NAME = DATABRICKS
|
33
32
|
REQUIREMENTS = ["databricks-sdk==0.28.0"]
|
34
33
|
|
34
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["numpy", "pandas"]
|
35
|
+
|
36
|
+
@classmethod
|
37
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
38
|
+
"""Method to get the requirements for the integration.
|
39
|
+
|
40
|
+
Args:
|
41
|
+
target_os: The target operating system to get the requirements for.
|
42
|
+
|
43
|
+
Returns:
|
44
|
+
A list of requirements.
|
45
|
+
"""
|
46
|
+
from zenml.integrations.numpy import NumpyIntegration
|
47
|
+
from zenml.integrations.pandas import PandasIntegration
|
48
|
+
|
49
|
+
return cls.REQUIREMENTS + \
|
50
|
+
NumpyIntegration.get_requirements(target_os=target_os) + \
|
51
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
52
|
+
|
35
53
|
@classmethod
|
36
54
|
def flavors(cls) -> List[Type[Flavor]]:
|
37
55
|
"""Declare the stack component flavors for the Databricks integration.
|
@@ -102,6 +102,15 @@ class DatabricksOrchestratorConfig(
|
|
102
102
|
"""
|
103
103
|
return True
|
104
104
|
|
105
|
+
@property
|
106
|
+
def is_schedulable(self) -> bool:
|
107
|
+
"""Whether the orchestrator is schedulable or not.
|
108
|
+
|
109
|
+
Returns:
|
110
|
+
Whether the orchestrator is schedulable or not.
|
111
|
+
"""
|
112
|
+
return True
|
113
|
+
|
105
114
|
|
106
115
|
class DatabricksOrchestratorFlavor(BaseOrchestratorFlavor):
|
107
116
|
"""Databricks orchestrator flavor."""
|
@@ -13,18 +13,17 @@
|
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Deepchecks integration for ZenML.
|
15
15
|
|
16
|
-
The Deepchecks integration provides a way to validate your data in your
|
17
|
-
It includes a way to detect data anomalies and define checks to
|
18
|
-
data.
|
16
|
+
The Deepchecks integration provides a way to validate your data in your
|
17
|
+
pipelines. It includes a way to detect data anomalies and define checks to
|
18
|
+
ensure quality of data.
|
19
19
|
|
20
20
|
The integration includes custom materializers to store and visualize Deepchecks
|
21
21
|
`SuiteResults`.
|
22
22
|
"""
|
23
23
|
|
24
|
-
from typing import List, Type
|
24
|
+
from typing import List, Type, Optional
|
25
25
|
|
26
|
-
from zenml.
|
27
|
-
from zenml.integrations.constants import DEEPCHECKS
|
26
|
+
from zenml.integrations.constants import DEEPCHECKS, PANDAS
|
28
27
|
from zenml.integrations.integration import Integration
|
29
28
|
from zenml.stack import Flavor
|
30
29
|
|
@@ -36,21 +35,40 @@ class DeepchecksIntegration(Integration):
|
|
36
35
|
|
37
36
|
NAME = DEEPCHECKS
|
38
37
|
REQUIREMENTS = [
|
39
|
-
"deepchecks[vision]
|
38
|
+
"deepchecks[vision]>=0.18.0",
|
40
39
|
"torchvision>=0.14.0",
|
41
|
-
"pandas<2.0.0",
|
42
40
|
"opencv-python==4.5.5.64", # pin to same version
|
43
41
|
"opencv-python-headless==4.5.5.64", # pin to same version
|
44
42
|
"tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
|
43
|
+
# The deepchecks integrations requires pandas to work.
|
44
|
+
# However, their version 0.18.0 is still not compatible with
|
45
|
+
# pandas>=2.2.0, so we limit the version here.
|
46
|
+
"pandas<2.2.0",
|
45
47
|
]
|
48
|
+
|
46
49
|
APT_PACKAGES = ["ffmpeg", "libsm6", "libxext6"]
|
47
|
-
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas","torchvision","tenacity"]
|
50
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas", "torchvision", "tenacity"]
|
48
51
|
|
49
|
-
@
|
50
|
-
def activate() -> None:
|
52
|
+
@classmethod
|
53
|
+
def activate(cls) -> None:
|
51
54
|
"""Activate the Deepchecks integration."""
|
52
55
|
from zenml.integrations.deepchecks import materializers # noqa
|
53
56
|
|
57
|
+
@classmethod
|
58
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
59
|
+
"""Method to get the requirements for the integration.
|
60
|
+
|
61
|
+
Args:
|
62
|
+
target_os: The target operating system to get the requirements for.
|
63
|
+
|
64
|
+
Returns:
|
65
|
+
A list of requirements.
|
66
|
+
"""
|
67
|
+
from zenml.integrations.pandas import PandasIntegration
|
68
|
+
|
69
|
+
return cls.REQUIREMENTS + \
|
70
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
71
|
+
|
54
72
|
@classmethod
|
55
73
|
def flavors(cls) -> List[Type[Flavor]]:
|
56
74
|
"""Declare the stack component flavors for the Deepchecks integration.
|
@@ -18,7 +18,9 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type
|
|
18
18
|
from deepchecks.tabular import Dataset
|
19
19
|
|
20
20
|
from zenml.enums import ArtifactType, VisualizationType
|
21
|
-
from zenml.materializers.pandas_materializer import
|
21
|
+
from zenml.integrations.pandas.materializers.pandas_materializer import (
|
22
|
+
PandasMaterializer,
|
23
|
+
)
|
22
24
|
|
23
25
|
if TYPE_CHECKING:
|
24
26
|
from zenml.metadata.metadata_types import MetadataType
|
@@ -176,9 +176,6 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck):
|
|
176
176
|
TABULAR_FEATURE_LABEL_CORRELATION = source_utils.resolve(
|
177
177
|
tabular_checks.FeatureLabelCorrelation
|
178
178
|
).import_path
|
179
|
-
TABULAR_IDENTIFIER_LEAKAGE = source_utils.resolve(
|
180
|
-
tabular_checks.IdentifierLeakage
|
181
|
-
).import_path
|
182
179
|
TABULAR_IS_SINGLE_VALUE = source_utils.resolve(
|
183
180
|
tabular_checks.IsSingleValue
|
184
181
|
).import_path
|
@@ -234,9 +231,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
|
|
234
231
|
TABULAR_DATE_TRAIN_TEST_LEAKAGE_OVERLAP = source_utils.resolve(
|
235
232
|
tabular_checks.DateTrainTestLeakageOverlap
|
236
233
|
).import_path
|
237
|
-
TABULAR_DOMINANT_FREQUENCY_CHANGE = source_utils.resolve(
|
238
|
-
tabular_checks.DominantFrequencyChange
|
239
|
-
).import_path
|
240
234
|
TABULAR_FEATURE_LABEL_CORRELATION_CHANGE = source_utils.resolve(
|
241
235
|
tabular_checks.FeatureLabelCorrelationChange
|
242
236
|
).import_path
|
@@ -262,9 +256,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
|
|
262
256
|
tabular_checks.WholeDatasetDrift
|
263
257
|
).import_path
|
264
258
|
|
265
|
-
VISION_FEATURE_LABEL_CORRELATION_CHANGE = source_utils.resolve(
|
266
|
-
vision_checks.FeatureLabelCorrelationChange
|
267
|
-
).import_path
|
268
259
|
VISION_HEATMAP_COMPARISON = source_utils.resolve(
|
269
260
|
vision_checks.HeatmapComparison
|
270
261
|
).import_path
|
@@ -277,9 +268,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
|
|
277
268
|
VISION_NEW_LABELS = source_utils.resolve(
|
278
269
|
vision_checks.NewLabels
|
279
270
|
).import_path
|
280
|
-
VISION_SIMILAR_IMAGE_LEAKAGE = source_utils.resolve(
|
281
|
-
vision_checks.SimilarImageLeakage
|
282
|
-
).import_path
|
283
271
|
VISION_TRAIN_TEST_LABEL_DRIFT = source_utils.resolve(
|
284
272
|
vision_checks.TrainTestLabelDrift
|
285
273
|
).import_path
|
@@ -324,21 +312,12 @@ class DeepchecksModelValidationCheck(DeepchecksValidationCheck):
|
|
324
312
|
VISION_CONFUSION_MATRIX_REPORT = source_utils.resolve(
|
325
313
|
vision_checks.ConfusionMatrixReport
|
326
314
|
).import_path
|
327
|
-
VISION_IMAGE_SEGMENT_PERFORMANCE = source_utils.resolve(
|
328
|
-
vision_checks.ImageSegmentPerformance
|
329
|
-
).import_path
|
330
315
|
VISION_MEAN_AVERAGE_PRECISION_REPORT = source_utils.resolve(
|
331
316
|
vision_checks.MeanAveragePrecisionReport
|
332
317
|
).import_path
|
333
318
|
VISION_MEAN_AVERAGE_RECALL_REPORT = source_utils.resolve(
|
334
319
|
vision_checks.MeanAverageRecallReport
|
335
320
|
).import_path
|
336
|
-
VISION_ROBUSTNESS_REPORT = source_utils.resolve(
|
337
|
-
vision_checks.RobustnessReport
|
338
|
-
).import_path
|
339
|
-
VISION_SINGLE_DATASET_SCALAR_PERFORMANCE = source_utils.resolve(
|
340
|
-
vision_checks.SingleDatasetScalarPerformance
|
341
|
-
).import_path
|
342
321
|
|
343
322
|
|
344
323
|
class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
|
@@ -358,12 +337,6 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
|
|
358
337
|
TABULAR_BOOSTING_OVERFIT = source_utils.resolve(
|
359
338
|
tabular_checks.BoostingOverfit
|
360
339
|
).import_path
|
361
|
-
TABULAR_MODEL_ERROR_ANALYSIS = source_utils.resolve(
|
362
|
-
tabular_checks.ModelErrorAnalysis
|
363
|
-
).import_path
|
364
|
-
TABULAR_PERFORMANCE_REPORT = source_utils.resolve(
|
365
|
-
tabular_checks.PerformanceReport
|
366
|
-
).import_path
|
367
340
|
TABULAR_SIMPLE_MODEL_COMPARISON = source_utils.resolve(
|
368
341
|
tabular_checks.SimpleModelComparison
|
369
342
|
).import_path
|
@@ -377,9 +350,6 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
|
|
377
350
|
VISION_CLASS_PERFORMANCE = source_utils.resolve(
|
378
351
|
vision_checks.ClassPerformance
|
379
352
|
).import_path
|
380
|
-
VISION_MODEL_ERROR_ANALYSIS = source_utils.resolve(
|
381
|
-
vision_checks.ModelErrorAnalysis
|
382
|
-
).import_path
|
383
353
|
VISION_SIMPLE_MODEL_COMPARISON = source_utils.resolve(
|
384
354
|
vision_checks.SimpleModelComparison
|
385
355
|
).import_path
|
@@ -25,7 +25,7 @@ file.
|
|
25
25
|
import logging
|
26
26
|
import os
|
27
27
|
import warnings
|
28
|
-
from typing import List, Type
|
28
|
+
from typing import List, Type, Optional
|
29
29
|
|
30
30
|
from zenml.integrations.constants import EVIDENTLY
|
31
31
|
from zenml.integrations.integration import Integration
|
@@ -57,7 +57,22 @@ class EvidentlyIntegration(Integration):
|
|
57
57
|
"evidently>=0.4.16,<=0.4.22",
|
58
58
|
"tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
|
59
59
|
]
|
60
|
-
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]
|
60
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity", "pandas"]
|
61
|
+
|
62
|
+
@classmethod
|
63
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
64
|
+
"""Method to get the requirements for the integration.
|
65
|
+
|
66
|
+
Args:
|
67
|
+
target_os: The target operating system to get the requirements for.
|
68
|
+
|
69
|
+
Returns:
|
70
|
+
A list of requirements.
|
71
|
+
"""
|
72
|
+
from zenml.integrations.pandas import PandasIntegration
|
73
|
+
|
74
|
+
return cls.REQUIREMENTS + \
|
75
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
61
76
|
|
62
77
|
@classmethod
|
63
78
|
def flavors(cls) -> List[Type[Flavor]]:
|
@@ -12,8 +12,8 @@
|
|
12
12
|
# or implied. See the License for the specific language governing
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Facets integration for ZenML."""
|
15
|
-
|
16
|
-
from zenml.integrations.constants import FACETS
|
15
|
+
from typing import Optional, List
|
16
|
+
from zenml.integrations.constants import FACETS, PANDAS
|
17
17
|
from zenml.integrations.integration import Integration
|
18
18
|
|
19
19
|
|
@@ -21,13 +21,29 @@ class FacetsIntegration(Integration):
|
|
21
21
|
"""Definition of Facets integration for ZenML."""
|
22
22
|
|
23
23
|
NAME = FACETS
|
24
|
-
REQUIREMENTS = ["facets-overview>=1.0.0"
|
24
|
+
REQUIREMENTS = ["facets-overview>=1.0.0"]
|
25
|
+
|
25
26
|
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
|
26
27
|
|
27
|
-
@
|
28
|
-
def activate() -> None:
|
28
|
+
@classmethod
|
29
|
+
def activate(cls) -> None:
|
29
30
|
"""Activate the Facets integration."""
|
30
31
|
from zenml.integrations.facets import materializers # noqa
|
31
32
|
|
33
|
+
@classmethod
|
34
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
35
|
+
"""Method to get the requirements for the integration.
|
36
|
+
|
37
|
+
Args:
|
38
|
+
target_os: The target operating system to get the requirements for.
|
39
|
+
|
40
|
+
Returns:
|
41
|
+
A list of requirements.
|
42
|
+
"""
|
43
|
+
from zenml.integrations.pandas import PandasIntegration
|
44
|
+
|
45
|
+
return cls.REQUIREMENTS + \
|
46
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
47
|
+
|
32
48
|
|
33
49
|
FacetsIntegration.check_installation()
|
@@ -17,9 +17,8 @@ The Feast integration offers a way to connect to a Feast Feature Store. ZenML
|
|
17
17
|
implements a dedicated stack component that you can access as part of your ZenML
|
18
18
|
steps in the usual ways.
|
19
19
|
"""
|
20
|
-
from typing import List, Type
|
20
|
+
from typing import List, Type, Optional
|
21
21
|
|
22
|
-
from zenml.enums import StackComponentType
|
23
22
|
from zenml.integrations.constants import FEAST
|
24
23
|
from zenml.integrations.integration import Integration
|
25
24
|
from zenml.stack import Flavor
|
@@ -31,10 +30,9 @@ class FeastIntegration(Integration):
|
|
31
30
|
"""Definition of Feast integration for ZenML."""
|
32
31
|
|
33
32
|
NAME = FEAST
|
34
|
-
# click is added to keep the feast click version
|
35
|
-
|
36
|
-
|
37
|
-
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click"]
|
33
|
+
# click is added to keep the feast click version in sync with ZenML's click
|
34
|
+
REQUIREMENTS = ["feast", "click>=8.0.1,<8.1.4"]
|
35
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click", "pandas"]
|
38
36
|
|
39
37
|
@classmethod
|
40
38
|
def flavors(cls) -> List[Type[Flavor]]:
|
@@ -47,5 +45,20 @@ class FeastIntegration(Integration):
|
|
47
45
|
|
48
46
|
return [FeastFeatureStoreFlavor]
|
49
47
|
|
48
|
+
@classmethod
|
49
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
50
|
+
"""Method to get the requirements for the integration.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
target_os: The target operating system to get the requirements for.
|
54
|
+
|
55
|
+
Returns:
|
56
|
+
A list of requirements.
|
57
|
+
"""
|
58
|
+
from zenml.integrations.pandas import PandasIntegration
|
59
|
+
|
60
|
+
return cls.REQUIREMENTS + \
|
61
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
62
|
+
|
50
63
|
|
51
64
|
FeastIntegration.check_installation()
|
@@ -55,8 +55,8 @@ class GcpIntegration(Integration):
|
|
55
55
|
]
|
56
56
|
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes","kfp"]
|
57
57
|
|
58
|
-
@
|
59
|
-
def activate() -> None:
|
58
|
+
@classmethod
|
59
|
+
def activate(cls) -> None:
|
60
60
|
"""Activate the GCP integration."""
|
61
61
|
from zenml.integrations.gcp import service_connectors # noqa
|
62
62
|
|
@@ -162,6 +162,15 @@ class VertexOrchestratorConfig(
|
|
162
162
|
"""
|
163
163
|
return self.synchronous
|
164
164
|
|
165
|
+
@property
|
166
|
+
def is_schedulable(self) -> bool:
|
167
|
+
"""Whether the orchestrator is schedulable or not.
|
168
|
+
|
169
|
+
Returns:
|
170
|
+
Whether the orchestrator is schedulable or not.
|
171
|
+
"""
|
172
|
+
return True
|
173
|
+
|
165
174
|
|
166
175
|
class VertexOrchestratorFlavor(BaseOrchestratorFlavor):
|
167
176
|
"""Vertex Orchestrator flavor."""
|
@@ -32,7 +32,16 @@
|
|
32
32
|
import os
|
33
33
|
import re
|
34
34
|
import types
|
35
|
-
from typing import
|
35
|
+
from typing import (
|
36
|
+
TYPE_CHECKING,
|
37
|
+
Any,
|
38
|
+
Dict,
|
39
|
+
List,
|
40
|
+
Optional,
|
41
|
+
Tuple,
|
42
|
+
Type,
|
43
|
+
cast,
|
44
|
+
)
|
36
45
|
from uuid import UUID
|
37
46
|
|
38
47
|
from google.api_core import exceptions as google_exceptions
|
@@ -17,9 +17,8 @@ The Great Expectations integration enables you to use Great Expectations as a
|
|
17
17
|
way of profiling and validating your data.
|
18
18
|
"""
|
19
19
|
|
20
|
-
from typing import List, Type
|
20
|
+
from typing import List, Type, Optional
|
21
21
|
|
22
|
-
from zenml.enums import StackComponentType
|
23
22
|
from zenml.integrations.constants import GREAT_EXPECTATIONS
|
24
23
|
from zenml.integrations.integration import Integration
|
25
24
|
from zenml.stack import Flavor
|
@@ -31,12 +30,12 @@ class GreatExpectationsIntegration(Integration):
|
|
31
30
|
"""Definition of Great Expectations integration for ZenML."""
|
32
31
|
|
33
32
|
NAME = GREAT_EXPECTATIONS
|
34
|
-
REQUIREMENTS = [
|
35
|
-
"great-expectations>=0.17.15,<1.0",
|
36
|
-
]
|
33
|
+
REQUIREMENTS = ["great-expectations>=0.17.15,<1.0"]
|
37
34
|
|
38
|
-
|
39
|
-
|
35
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
|
36
|
+
|
37
|
+
@classmethod
|
38
|
+
def activate(cls) -> None:
|
40
39
|
"""Activate the Great Expectations integration."""
|
41
40
|
from zenml.integrations.great_expectations import materializers # noqa
|
42
41
|
|
@@ -53,5 +52,20 @@ class GreatExpectationsIntegration(Integration):
|
|
53
52
|
|
54
53
|
return [GreatExpectationsDataValidatorFlavor]
|
55
54
|
|
55
|
+
@classmethod
|
56
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
57
|
+
"""Method to get the requirements for the integration.
|
58
|
+
|
59
|
+
Args:
|
60
|
+
target_os: The target operating system to get the requirements for.
|
61
|
+
|
62
|
+
Returns:
|
63
|
+
A list of requirements.
|
64
|
+
"""
|
65
|
+
from zenml.integrations.pandas import PandasIntegration
|
66
|
+
|
67
|
+
return cls.REQUIREMENTS + \
|
68
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
69
|
+
|
56
70
|
|
57
71
|
GreatExpectationsIntegration.check_installation()
|
@@ -12,7 +12,8 @@
|
|
12
12
|
# or implied. See the License for the specific language governing
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
"""Initialization of the Huggingface integration."""
|
15
|
-
|
15
|
+
import sys
|
16
|
+
from typing import List, Type, Optional
|
16
17
|
|
17
18
|
from zenml.integrations.constants import HUGGINGFACE
|
18
19
|
from zenml.integrations.integration import Integration
|
@@ -26,20 +27,8 @@ class HuggingfaceIntegration(Integration):
|
|
26
27
|
"""Definition of Huggingface integration for ZenML."""
|
27
28
|
|
28
29
|
NAME = HUGGINGFACE
|
29
|
-
|
30
|
-
|
31
|
-
"datasets",
|
32
|
-
"huggingface_hub>0.19.0",
|
33
|
-
"accelerate",
|
34
|
-
"bitsandbytes>=0.41.3",
|
35
|
-
"peft",
|
36
|
-
# temporary fix for CI issue similar to:
|
37
|
-
# - https://github.com/huggingface/datasets/issues/6737
|
38
|
-
# - https://github.com/huggingface/datasets/issues/6697
|
39
|
-
# TODO try relaxing it back going forward
|
40
|
-
"fsspec<=2023.12.0",
|
41
|
-
]
|
42
|
-
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["fsspec"]
|
30
|
+
|
31
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["fsspec", "pandas"]
|
43
32
|
|
44
33
|
@classmethod
|
45
34
|
def activate(cls) -> None:
|
@@ -47,6 +36,41 @@ class HuggingfaceIntegration(Integration):
|
|
47
36
|
from zenml.integrations.huggingface import materializers # noqa
|
48
37
|
from zenml.integrations.huggingface import services
|
49
38
|
|
39
|
+
@classmethod
|
40
|
+
def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
|
41
|
+
"""Defines platform specific requirements for the integration.
|
42
|
+
|
43
|
+
Args:
|
44
|
+
target_os: The target operating system.
|
45
|
+
|
46
|
+
Returns:
|
47
|
+
A list of requirements.
|
48
|
+
"""
|
49
|
+
requirements = [
|
50
|
+
"datasets",
|
51
|
+
"huggingface_hub>0.19.0",
|
52
|
+
"accelerate",
|
53
|
+
"bitsandbytes>=0.41.3",
|
54
|
+
"peft",
|
55
|
+
# temporary fix for CI issue similar to:
|
56
|
+
# - https://github.com/huggingface/datasets/issues/6737
|
57
|
+
# - https://github.com/huggingface/datasets/issues/6697
|
58
|
+
# TODO try relaxing it back going forward
|
59
|
+
"fsspec<=2023.12.0",
|
60
|
+
]
|
61
|
+
|
62
|
+
# In python 3.8 higher transformers version lead to other packages breaking
|
63
|
+
if sys.version_info.minor > 8:
|
64
|
+
requirements += ["transformers"]
|
65
|
+
else:
|
66
|
+
requirements += ["transformers<=4.31"]
|
67
|
+
|
68
|
+
# Add the pandas integration requirements
|
69
|
+
from zenml.integrations.pandas import PandasIntegration
|
70
|
+
|
71
|
+
return requirements + \
|
72
|
+
PandasIntegration.get_requirements(target_os=target_os)
|
73
|
+
|
50
74
|
@classmethod
|
51
75
|
def flavors(cls) -> List[Type[Flavor]]:
|
52
76
|
"""Declare the stack component flavors for the Huggingface integration.
|
@@ -25,3 +25,6 @@ from zenml.integrations.huggingface.materializers.huggingface_tf_model_materiali
|
|
25
25
|
from zenml.integrations.huggingface.materializers.huggingface_tokenizer_materializer import (
|
26
26
|
HFTokenizerMaterializer,
|
27
27
|
)
|
28
|
+
from zenml.integrations.huggingface.materializers.huggingface_t5_materializer import (
|
29
|
+
HFT5Materializer,
|
30
|
+
)
|
@@ -31,9 +31,11 @@ from datasets import Dataset, load_from_disk
|
|
31
31
|
from datasets.dataset_dict import DatasetDict
|
32
32
|
|
33
33
|
from zenml.enums import ArtifactType, VisualizationType
|
34
|
+
from zenml.integrations.pandas.materializers.pandas_materializer import (
|
35
|
+
PandasMaterializer,
|
36
|
+
)
|
34
37
|
from zenml.io import fileio
|
35
38
|
from zenml.materializers.base_materializer import BaseMaterializer
|
36
|
-
from zenml.materializers.pandas_materializer import PandasMaterializer
|
37
39
|
from zenml.utils import io_utils
|
38
40
|
|
39
41
|
if TYPE_CHECKING:
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Copyright (c) ZenML GmbH 2024. All Rights Reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at:
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
12
|
+
# or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
"""Implementation of the Huggingface t5 materializer."""
|
15
|
+
|
16
|
+
import os
|
17
|
+
import tempfile
|
18
|
+
from typing import Any, ClassVar, Type, Union
|
19
|
+
|
20
|
+
from transformers import (
|
21
|
+
T5ForConditionalGeneration,
|
22
|
+
T5Tokenizer,
|
23
|
+
T5TokenizerFast,
|
24
|
+
)
|
25
|
+
|
26
|
+
from zenml.io import fileio
|
27
|
+
from zenml.materializers.base_materializer import BaseMaterializer
|
28
|
+
|
29
|
+
|
30
|
+
class HFT5Materializer(BaseMaterializer):
|
31
|
+
"""Base class for huggingface t5 models."""
|
32
|
+
|
33
|
+
SKIP_REGISTRATION: ClassVar[bool] = False
|
34
|
+
ASSOCIATED_TYPES = (
|
35
|
+
T5ForConditionalGeneration,
|
36
|
+
T5Tokenizer,
|
37
|
+
T5TokenizerFast,
|
38
|
+
)
|
39
|
+
|
40
|
+
def load(
|
41
|
+
self, data_type: Type[Any]
|
42
|
+
) -> Union[T5ForConditionalGeneration, T5Tokenizer, T5TokenizerFast]:
|
43
|
+
"""Reads a T5ForConditionalGeneration model or T5Tokenizer from a serialized zip file.
|
44
|
+
|
45
|
+
Args:
|
46
|
+
data_type: A T5ForConditionalGeneration or T5Tokenizer type.
|
47
|
+
|
48
|
+
Returns:
|
49
|
+
A T5ForConditionalGeneration or T5Tokenizer object.
|
50
|
+
|
51
|
+
Raises:
|
52
|
+
ValueError: Unsupported data type used
|
53
|
+
"""
|
54
|
+
filepath = self.uri
|
55
|
+
|
56
|
+
with tempfile.TemporaryDirectory(prefix="zenml-temp-") as temp_dir:
|
57
|
+
# Copy files from artifact store to temporary directory
|
58
|
+
for file in fileio.listdir(filepath):
|
59
|
+
src = os.path.join(filepath, file)
|
60
|
+
dst = os.path.join(temp_dir, file)
|
61
|
+
if fileio.isdir(src):
|
62
|
+
fileio.makedirs(dst)
|
63
|
+
for subfile in fileio.listdir(src):
|
64
|
+
subsrc = os.path.join(src, subfile)
|
65
|
+
subdst = os.path.join(dst, subfile)
|
66
|
+
fileio.copy(subsrc, subdst)
|
67
|
+
else:
|
68
|
+
fileio.copy(src, dst)
|
69
|
+
|
70
|
+
# Load the model or tokenizer from the temporary directory
|
71
|
+
if data_type in [
|
72
|
+
T5ForConditionalGeneration,
|
73
|
+
T5Tokenizer,
|
74
|
+
T5TokenizerFast,
|
75
|
+
]:
|
76
|
+
return data_type.from_pretrained(temp_dir)
|
77
|
+
else:
|
78
|
+
raise ValueError(f"Unsupported data type: {data_type}")
|
79
|
+
|
80
|
+
def save(
|
81
|
+
self,
|
82
|
+
obj: Union[T5ForConditionalGeneration, T5Tokenizer, T5TokenizerFast],
|
83
|
+
) -> None:
|
84
|
+
"""Creates a serialization for a T5ForConditionalGeneration model or T5Tokenizer.
|
85
|
+
|
86
|
+
Args:
|
87
|
+
obj: A T5ForConditionalGeneration model or T5Tokenizer.
|
88
|
+
"""
|
89
|
+
# Create a temporary directory
|
90
|
+
with tempfile.TemporaryDirectory(prefix="zenml-temp-") as temp_dir:
|
91
|
+
# Save the model or tokenizer
|
92
|
+
obj.save_pretrained(temp_dir)
|
93
|
+
|
94
|
+
# Copy the directory to the artifact store
|
95
|
+
filepath = self.uri
|
96
|
+
fileio.makedirs(filepath)
|
97
|
+
for file in os.listdir(temp_dir):
|
98
|
+
src = os.path.join(temp_dir, file)
|
99
|
+
dst = os.path.join(filepath, file)
|
100
|
+
if os.path.isdir(src):
|
101
|
+
fileio.makedirs(dst)
|
102
|
+
for subfile in os.listdir(src):
|
103
|
+
subsrc = os.path.join(src, subfile)
|
104
|
+
subdst = os.path.join(dst, subfile)
|
105
|
+
fileio.copy(subsrc, subdst)
|
106
|
+
else:
|
107
|
+
fileio.copy(src, dst)
|