zenml-nightly 0.64.0.dev20240811__py3-none-any.whl → 0.66.0.dev20240910__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. README.md +1 -1
  2. RELEASE_NOTES.md +126 -4
  3. zenml/VERSION +1 -1
  4. zenml/artifacts/utils.py +13 -6
  5. zenml/cli/__init__.py +1 -1
  6. zenml/cli/base.py +4 -4
  7. zenml/cli/integration.py +48 -9
  8. zenml/cli/pipeline.py +9 -2
  9. zenml/cli/stack.py +39 -27
  10. zenml/cli/utils.py +13 -0
  11. zenml/client.py +15 -17
  12. zenml/config/compiler.py +34 -0
  13. zenml/config/server_config.py +30 -0
  14. zenml/config/source.py +3 -7
  15. zenml/constants.py +5 -3
  16. zenml/entrypoints/base_entrypoint_configuration.py +41 -27
  17. zenml/entrypoints/step_entrypoint_configuration.py +5 -2
  18. zenml/enums.py +2 -0
  19. zenml/environment.py +31 -0
  20. zenml/feature_stores/base_feature_store.py +4 -6
  21. zenml/integrations/__init__.py +3 -0
  22. zenml/integrations/airflow/flavors/airflow_orchestrator_flavor.py +9 -0
  23. zenml/integrations/aws/__init__.py +2 -2
  24. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +2 -1
  25. zenml/integrations/azure/__init__.py +2 -2
  26. zenml/integrations/azure/azureml_utils.py +201 -0
  27. zenml/integrations/azure/flavors/azureml.py +139 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +20 -118
  29. zenml/integrations/azure/flavors/azureml_step_operator_flavor.py +67 -14
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +58 -172
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +1 -0
  32. zenml/integrations/azure/service_connectors/azure_service_connector.py +4 -0
  33. zenml/integrations/azure/step_operators/azureml_step_operator.py +78 -177
  34. zenml/integrations/constants.py +3 -0
  35. zenml/integrations/databricks/__init__.py +22 -4
  36. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +9 -0
  37. zenml/integrations/deepchecks/__init__.py +29 -11
  38. zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py +3 -1
  39. zenml/integrations/deepchecks/validation_checks.py +0 -30
  40. zenml/integrations/evidently/__init__.py +17 -2
  41. zenml/integrations/facets/__init__.py +21 -5
  42. zenml/integrations/feast/__init__.py +19 -6
  43. zenml/integrations/gcp/__init__.py +2 -2
  44. zenml/integrations/gcp/flavors/vertex_orchestrator_flavor.py +9 -0
  45. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +10 -1
  46. zenml/integrations/great_expectations/__init__.py +21 -7
  47. zenml/integrations/huggingface/__init__.py +39 -15
  48. zenml/integrations/huggingface/materializers/__init__.py +3 -0
  49. zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py +3 -1
  50. zenml/integrations/huggingface/materializers/huggingface_pt_model_materializer.py +1 -1
  51. zenml/integrations/huggingface/materializers/huggingface_t5_materializer.py +107 -0
  52. zenml/integrations/huggingface/materializers/huggingface_tf_model_materializer.py +1 -1
  53. zenml/integrations/huggingface/materializers/huggingface_tokenizer_materializer.py +2 -2
  54. zenml/integrations/huggingface/steps/accelerate_runner.py +108 -85
  55. zenml/integrations/hyperai/flavors/hyperai_orchestrator_flavor.py +9 -0
  56. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +9 -0
  57. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +10 -1
  58. zenml/integrations/kubernetes/flavors/kubernetes_orchestrator_flavor.py +9 -0
  59. zenml/integrations/kubernetes/orchestrators/kubernetes_orchestrator.py +10 -1
  60. zenml/integrations/lightning/__init__.py +48 -0
  61. zenml/integrations/lightning/flavors/__init__.py +23 -0
  62. zenml/integrations/lightning/flavors/lightning_orchestrator_flavor.py +148 -0
  63. zenml/integrations/lightning/orchestrators/__init__.py +23 -0
  64. zenml/integrations/lightning/orchestrators/lightning_orchestrator.py +596 -0
  65. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint.py +307 -0
  66. zenml/integrations/lightning/orchestrators/lightning_orchestrator_entrypoint_configuration.py +77 -0
  67. zenml/integrations/lightning/orchestrators/utils.py +67 -0
  68. zenml/integrations/mlflow/__init__.py +43 -5
  69. zenml/integrations/mlflow/services/mlflow_deployment.py +26 -0
  70. zenml/integrations/numpy/__init__.py +32 -0
  71. zenml/integrations/numpy/materializers/__init__.py +18 -0
  72. zenml/integrations/numpy/materializers/numpy_materializer.py +246 -0
  73. zenml/integrations/pandas/__init__.py +32 -0
  74. zenml/integrations/pandas/materializers/__init__.py +18 -0
  75. zenml/integrations/pandas/materializers/pandas_materializer.py +192 -0
  76. zenml/integrations/prodigy/annotators/prodigy_annotator.py +1 -1
  77. zenml/integrations/seldon/__init__.py +18 -3
  78. zenml/integrations/sklearn/__init__.py +1 -1
  79. zenml/integrations/skypilot_azure/__init__.py +1 -1
  80. zenml/integrations/tensorboard/__init__.py +1 -1
  81. zenml/integrations/tensorflow/__init__.py +2 -2
  82. zenml/integrations/tensorflow/materializers/tf_dataset_materializer.py +2 -2
  83. zenml/integrations/whylogs/__init__.py +18 -2
  84. zenml/logging/step_logging.py +9 -2
  85. zenml/materializers/__init__.py +0 -4
  86. zenml/materializers/base_materializer.py +4 -0
  87. zenml/materializers/numpy_materializer.py +23 -234
  88. zenml/materializers/pandas_materializer.py +22 -179
  89. zenml/model/model.py +91 -2
  90. zenml/model/utils.py +5 -5
  91. zenml/models/__init__.py +16 -3
  92. zenml/models/v2/core/model_version.py +1 -1
  93. zenml/models/v2/core/pipeline_run.py +31 -1
  94. zenml/models/v2/core/stack.py +51 -20
  95. zenml/models/v2/core/step_run.py +28 -0
  96. zenml/models/v2/misc/info_models.py +78 -0
  97. zenml/new/pipelines/pipeline.py +65 -25
  98. zenml/new/pipelines/run_utils.py +57 -136
  99. zenml/new/steps/step_context.py +17 -6
  100. zenml/orchestrators/base_orchestrator.py +9 -0
  101. zenml/orchestrators/step_launcher.py +37 -14
  102. zenml/orchestrators/step_runner.py +14 -13
  103. zenml/orchestrators/utils.py +107 -7
  104. zenml/service_connectors/service_connector_utils.py +2 -2
  105. zenml/stack/utils.py +11 -2
  106. zenml/stack_deployments/azure_stack_deployment.py +2 -1
  107. zenml/steps/base_step.py +62 -25
  108. zenml/steps/utils.py +115 -3
  109. zenml/utils/cloud_utils.py +8 -8
  110. zenml/utils/code_utils.py +130 -32
  111. zenml/utils/function_utils.py +7 -7
  112. zenml/utils/notebook_utils.py +14 -0
  113. zenml/utils/pipeline_docker_image_builder.py +1 -11
  114. zenml/utils/pydantic_utils.py +3 -3
  115. zenml/utils/secret_utils.py +2 -2
  116. zenml/utils/settings_utils.py +1 -1
  117. zenml/utils/source_utils.py +67 -21
  118. zenml/utils/string_utils.py +29 -0
  119. zenml/zen_server/dashboard/assets/{404-CRAA_Lew.js → 404-iO8vpun1.js} +1 -1
  120. zenml/zen_server/dashboard/assets/{@radix-BXWm7HOa.js → @radix-DnFH_oo1.js} +1 -1
  121. zenml/zen_server/dashboard/assets/{@react-router-l3lMcXA2.js → @react-router-APVeuk-U.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{@reactflow-CeVxyqYT.js → @reactflow-B6kq9fJZ.js} +2 -2
  123. zenml/zen_server/dashboard/assets/{@tanstack-FmcYZMuX.js → @tanstack-QbMbTrh5.js} +1 -1
  124. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-BXeSvmMY.js +1 -0
  125. zenml/zen_server/dashboard/assets/{CodeSnippet-D0VLxT2A.js → CodeSnippet-DNWdQmbo.js} +2 -2
  126. zenml/zen_server/dashboard/assets/CollapsibleCard-B2OVjWYE.js +1 -0
  127. zenml/zen_server/dashboard/assets/Commands-DsoaVElZ.js +1 -0
  128. zenml/zen_server/dashboard/assets/CopyButton-BqE_-PHO.js +2 -0
  129. zenml/zen_server/dashboard/assets/{CsvVizualization-D3kAypDj.js → CsvVizualization-Dyasr2jU.js} +6 -6
  130. zenml/zen_server/dashboard/assets/{edit-C0MVvPD2.js → DialogItem-Cz1VLRwa.js} +1 -1
  131. zenml/zen_server/dashboard/assets/{DisplayDate-DizbSeT-.js → DisplayDate-DkCy54Bp.js} +1 -1
  132. zenml/zen_server/dashboard/assets/EditSecretDialog-Du423_3U.js +1 -0
  133. zenml/zen_server/dashboard/assets/{EmptyState-BHblM39I.js → EmptyState-Cs3DEmso.js} +1 -1
  134. zenml/zen_server/dashboard/assets/{Error-C6LeJSER.js → Error-DorJD_va.js} +1 -1
  135. zenml/zen_server/dashboard/assets/ExecutionStatus-CIfQTutR.js +1 -0
  136. zenml/zen_server/dashboard/assets/{Helpbox-aAB2XP-z.js → Helpbox-CmfvtNeq.js} +1 -1
  137. zenml/zen_server/dashboard/assets/Infobox-BL9NOS37.js +1 -0
  138. zenml/zen_server/dashboard/assets/{InlineAvatar-DpTLgM3Q.js → InlineAvatar-Ds2ZFHPc.js} +1 -1
  139. zenml/zen_server/dashboard/assets/{Lock-CNyJvf2r.js → Lock-CmIn0szs.js} +1 -1
  140. zenml/zen_server/dashboard/assets/{MarkdownVisualization-Bajxn0HY.js → MarkdownVisualization-DS05sfBm.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{NumberBox-BmKE0qnO.js → NumberBox-CrN0_kqI.js} +1 -1
  142. zenml/zen_server/dashboard/assets/Partials-DX-8iEa1.js +1 -0
  143. zenml/zen_server/dashboard/assets/{PasswordChecker-yGGoJSB-.js → PasswordChecker-DE71J_3F.js} +1 -1
  144. zenml/zen_server/dashboard/assets/ProviderIcon-BOQJgapd.js +1 -0
  145. zenml/zen_server/dashboard/assets/ProviderRadio-BsYBw9YA.js +1 -0
  146. zenml/zen_server/dashboard/assets/SearchField-W3GXpLlI.js +1 -0
  147. zenml/zen_server/dashboard/assets/SetPassword-B-0a8UCj.js +1 -0
  148. zenml/zen_server/dashboard/assets/{Tick-uxv80Q6a.js → Tick-i1DYsVcX.js} +1 -1
  149. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-oN4G3sKz.js → UpdatePasswordSchemas-C6Zb7ASL.js} +1 -1
  150. zenml/zen_server/dashboard/assets/UsageReason-CCnzmwS8.js +1 -0
  151. zenml/zen_server/dashboard/assets/WizardFooter-BHbO7zOa.js +1 -0
  152. zenml/zen_server/dashboard/assets/all-pipeline-runs-query-BBEe6I9-.js +1 -0
  153. zenml/zen_server/dashboard/assets/{check-circle-1_I207rW.js → check-circle-DOoS4yhF.js} +1 -1
  154. zenml/zen_server/dashboard/assets/{chevron-down-BpaF8JqM.js → chevron-down-Cwb-W_B_.js} +1 -1
  155. zenml/zen_server/dashboard/assets/{chevron-right-double-Dk8e2L99.js → chevron-right-double-c9H46Kl8.js} +1 -1
  156. zenml/zen_server/dashboard/assets/{cloud-only-BkUuI0lZ.js → cloud-only-BuP4Kt_7.js} +1 -1
  157. zenml/zen_server/dashboard/assets/code-browser-BJYErIjr.js +1 -0
  158. zenml/zen_server/dashboard/assets/codespaces-BitYDX9d.gif +0 -0
  159. zenml/zen_server/dashboard/assets/{copy-f3XGPPxt.js → copy-CaGlDsUy.js} +1 -1
  160. zenml/zen_server/dashboard/assets/create-stack-B2x2d4r1.js +1 -0
  161. zenml/zen_server/dashboard/assets/{docker-8uj__HHK.js → docker-BFAFXr2_.js} +1 -1
  162. zenml/zen_server/dashboard/assets/{dots-horizontal-sKQlWEni.js → dots-horizontal-C6K59vUm.js} +1 -1
  163. zenml/zen_server/dashboard/assets/flyte-Cj-xy_8I.svg +10 -0
  164. zenml/zen_server/dashboard/assets/form-schemas-Bap0f854.js +1 -0
  165. zenml/zen_server/dashboard/assets/gcp-Dj6ntk0L.js +1 -0
  166. zenml/zen_server/dashboard/assets/{help-FuHlZwn0.js → help-CwN931fX.js} +1 -1
  167. zenml/zen_server/dashboard/assets/{index-Bd1xgUQG.js → index-5GJ5ysEZ.js} +1 -1
  168. zenml/zen_server/dashboard/assets/{index-DaGknux4.css → index-6DYjZgDn.css} +1 -1
  169. zenml/zen_server/dashboard/assets/index-B9wVwe7u.js +55 -0
  170. zenml/zen_server/dashboard/assets/index-DFi8BroH.js +1 -0
  171. zenml/zen_server/dashboard/assets/{index.esm-DT4uyn2i.js → index.esm-BE1uqCX5.js} +1 -1
  172. zenml/zen_server/dashboard/assets/kubernetes-BjbR6D-1.js +1 -0
  173. zenml/zen_server/dashboard/assets/{layout-D6oiSbfd.js → layout-Dru15_XR.js} +1 -1
  174. zenml/zen_server/dashboard/assets/link-external-BT2L8hAQ.js +1 -0
  175. zenml/zen_server/dashboard/assets/{login-mutation-13A_JSVA.js → login-mutation-DwxUz8VA.js} +1 -1
  176. zenml/zen_server/dashboard/assets/{logs-CgeE2vZP.js → logs-GiDJXbLS.js} +1 -1
  177. zenml/zen_server/dashboard/assets/metaflow-weOkWNyT.svg +10 -0
  178. zenml/zen_server/dashboard/assets/{not-found-B0Mmb90p.js → not-found-D5i9DunU.js} +1 -1
  179. zenml/zen_server/dashboard/assets/{package-DdkziX79.js → package-DYKZ5jKW.js} +1 -1
  180. zenml/zen_server/dashboard/assets/page-BFuJICXM.js +9 -0
  181. zenml/zen_server/dashboard/assets/{page-BGwA9B1M.js → page-BiF8hLbO.js} +1 -1
  182. zenml/zen_server/dashboard/assets/{page-DugsjcQ_.js → page-BitfWsiW.js} +1 -1
  183. zenml/zen_server/dashboard/assets/page-CDOQLrPC.js +1 -0
  184. zenml/zen_server/dashboard/assets/page-CEJWu1YO.js +1 -0
  185. zenml/zen_server/dashboard/assets/page-CIbehp7V.js +1 -0
  186. zenml/zen_server/dashboard/assets/page-CLiRGfWo.js +1 -0
  187. zenml/zen_server/dashboard/assets/page-CV44mQn9.js +1 -0
  188. zenml/zen_server/dashboard/assets/page-CrSdkteO.js +2 -0
  189. zenml/zen_server/dashboard/assets/page-D5F3DJjm.js +1 -0
  190. zenml/zen_server/dashboard/assets/page-DE03uZZR.js +1 -0
  191. zenml/zen_server/dashboard/assets/page-DFCK65G9.js +1 -0
  192. zenml/zen_server/dashboard/assets/{page-RnG-qhv9.js → page-DGMa3ZQL.js} +1 -1
  193. zenml/zen_server/dashboard/assets/page-DI-qTWrm.js +1 -0
  194. zenml/zen_server/dashboard/assets/page-DQGCHKrQ.js +1 -0
  195. zenml/zen_server/dashboard/assets/{page-DSTQnBk-.js → page-DQdwZZ9x.js} +1 -1
  196. zenml/zen_server/dashboard/assets/page-DgM-N9RL.js +1 -0
  197. zenml/zen_server/dashboard/assets/page-Dt8VgzbE.js +1 -0
  198. zenml/zen_server/dashboard/assets/{page-DLpOnf7u.js → page-J0s8Sq3N.js} +1 -1
  199. zenml/zen_server/dashboard/assets/page-WCQ659by.js +1 -0
  200. zenml/zen_server/dashboard/assets/page-bimkItOg.js +1 -0
  201. zenml/zen_server/dashboard/assets/{page-hQaiQXfg.js → page-iwoJnwPv.js} +1 -1
  202. zenml/zen_server/dashboard/assets/{page-YiF_fNbe.js → page-oS4hqS8M.js} +1 -1
  203. zenml/zen_server/dashboard/assets/page-oSqx9dkH.js +1 -0
  204. zenml/zen_server/dashboard/assets/page-p3GqEAUW.js +1 -0
  205. zenml/zen_server/dashboard/assets/page-qvcUVPE-.js +1 -0
  206. zenml/zen_server/dashboard/assets/page-xQG6GmFJ.js +1 -0
  207. zenml/zen_server/dashboard/assets/{persist-3-5nOJ6m.js → persist-mEZN_fgH.js} +1 -1
  208. zenml/zen_server/dashboard/assets/persist-xsYgVtR1.js +1 -0
  209. zenml/zen_server/dashboard/assets/{plus-FB9-lEq_.js → plus-Bc8eLSDM.js} +1 -1
  210. zenml/zen_server/dashboard/assets/{refresh-COb6KYDi.js → refresh-hfgWPeto.js} +1 -1
  211. zenml/zen_server/dashboard/assets/rocket-SESCGQXm.js +1 -0
  212. zenml/zen_server/dashboard/assets/sharedSchema-BfZcy7aP.js +14 -0
  213. zenml/zen_server/dashboard/assets/stack-detail-query-CU4egfhp.js +1 -0
  214. zenml/zen_server/dashboard/assets/templates-1S_8WeSK.webp +0 -0
  215. zenml/zen_server/dashboard/assets/{trash-Cd5CSFqA.js → trash-DUWZWzse.js} +1 -1
  216. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-B8GB_ubU.js → update-server-settings-mutation-DNqmQXDM.js} +1 -1
  217. zenml/zen_server/dashboard/assets/{url-hcMJkz8p.js → url-DwbuKk1b.js} +1 -1
  218. zenml/zen_server/dashboard/assets/{zod-CnykDKJj.js → zod-uFd1wBcd.js} +1 -1
  219. zenml/zen_server/dashboard/index.html +7 -7
  220. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  221. zenml/zen_server/dashboard_legacy/index.html +1 -1
  222. zenml/zen_server/dashboard_legacy/{precache-manifest.9c473c96a43298343a7ce1256183123b.js → precache-manifest.290b95d5b43efa3368b3dc63d20c4782.js} +4 -4
  223. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  224. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js → main.840d1bf0.chunk.js} +2 -2
  225. zenml/zen_server/dashboard_legacy/static/js/{main.463c90b9.chunk.js.map → main.840d1bf0.chunk.js.map} +1 -1
  226. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  227. zenml/zen_server/deploy/helm/README.md +2 -2
  228. zenml/zen_server/routers/service_connectors_endpoints.py +2 -4
  229. zenml/zen_server/routers/workspaces_endpoints.py +20 -66
  230. zenml/zen_server/secure_headers.py +120 -0
  231. zenml/zen_server/template_execution/runner_entrypoint_configuration.py +0 -2
  232. zenml/zen_server/template_execution/utils.py +1 -0
  233. zenml/zen_server/utils.py +0 -100
  234. zenml/zen_server/zen_server_api.py +4 -2
  235. zenml/zen_stores/migrations/versions/0.65.0_release.py +23 -0
  236. zenml/zen_stores/migrations/versions/0.66.0_release.py +23 -0
  237. zenml/zen_stores/migrations/versions/bf2120261b5a_add_configured_model_version_id.py +74 -0
  238. zenml/zen_stores/rest_zen_store.py +4 -21
  239. zenml/zen_stores/schemas/constants.py +16 -0
  240. zenml/zen_stores/schemas/model_schemas.py +9 -3
  241. zenml/zen_stores/schemas/pipeline_run_schemas.py +22 -8
  242. zenml/zen_stores/schemas/step_run_schemas.py +23 -12
  243. zenml/zen_stores/sql_zen_store.py +312 -300
  244. zenml/zen_stores/zen_store_interface.py +0 -16
  245. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/METADATA +10 -12
  246. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/RECORD +249 -217
  247. zenml/models/v2/misc/full_stack.py +0 -129
  248. zenml/new/pipelines/model_utils.py +0 -72
  249. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +0 -1
  250. zenml/zen_server/dashboard/assets/AwarenessChannel-CLXo5rKM.js +0 -1
  251. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +0 -1
  252. zenml/zen_server/dashboard/assets/Commands-JrcZK-3j.js +0 -1
  253. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +0 -2
  254. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +0 -1
  255. zenml/zen_server/dashboard/assets/ExecutionStatus-jH4OrWBq.js +0 -1
  256. zenml/zen_server/dashboard/assets/Infobox-BQ0aty32.js +0 -1
  257. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +0 -1
  258. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +0 -1
  259. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +0 -1
  260. zenml/zen_server/dashboard/assets/SetPassword-52sNxNiO.js +0 -1
  261. zenml/zen_server/dashboard/assets/SuccessStep-DlkItqYG.js +0 -1
  262. zenml/zen_server/dashboard/assets/aws-0_3UsPif.js +0 -1
  263. zenml/zen_server/dashboard/assets/database-cXYNX9tt.js +0 -1
  264. zenml/zen_server/dashboard/assets/file-text-B9JibxTs.js +0 -1
  265. zenml/zen_server/dashboard/assets/index-DhIZtpxB.js +0 -55
  266. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +0 -1
  267. zenml/zen_server/dashboard/assets/page-B3ozwdD1.js +0 -1
  268. zenml/zen_server/dashboard/assets/page-BkjAUyTA.js +0 -1
  269. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +0 -1
  270. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +0 -2
  271. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +0 -1
  272. zenml/zen_server/dashboard/assets/page-C9kudd44.js +0 -9
  273. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +0 -1
  274. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +0 -1
  275. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +0 -1
  276. zenml/zen_server/dashboard/assets/page-Cgn-6v2Y.js +0 -1
  277. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +0 -1
  278. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +0 -1
  279. zenml/zen_server/dashboard/assets/page-DTysUGOy.js +0 -1
  280. zenml/zen_server/dashboard/assets/page-D_EXUFJb.js +0 -1
  281. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +0 -1
  282. zenml/zen_server/dashboard/assets/page-OFKSPyN7.js +0 -1
  283. zenml/zen_server/dashboard/assets/page-T2BtjwPl.js +0 -1
  284. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +0 -1
  285. zenml/zen_server/dashboard/assets/play-circle-XSkLR12B.js +0 -1
  286. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +0 -14
  287. zenml/zen_server/dashboard/assets/stack-detail-query-B-US_-wa.js +0 -1
  288. zenml/zen_server/dashboard/assets/terminal-grtjrIEJ.js +0 -1
  289. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/LICENSE +0 -0
  290. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/WHEEL +0 -0
  291. {zenml_nightly-0.64.0.dev20240811.dist-info → zenml_nightly-0.66.0.dev20240910.dist-info}/entry_points.txt +0 -0
@@ -13,11 +13,10 @@
13
13
  # permissions and limitations under the License.
14
14
  """Initialization of the Databricks integration for ZenML."""
15
15
 
16
- from typing import List, Type
16
+ from typing import List, Type, Optional
17
+
18
+ from zenml.integrations.constants import DATABRICKS
17
19
 
18
- from zenml.integrations.constants import (
19
- DATABRICKS,
20
- )
21
20
  from zenml.integrations.integration import Integration
22
21
  from zenml.stack import Flavor
23
22
 
@@ -32,6 +31,25 @@ class DatabricksIntegration(Integration):
32
31
  NAME = DATABRICKS
33
32
  REQUIREMENTS = ["databricks-sdk==0.28.0"]
34
33
 
34
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["numpy", "pandas"]
35
+
36
+ @classmethod
37
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
38
+ """Method to get the requirements for the integration.
39
+
40
+ Args:
41
+ target_os: The target operating system to get the requirements for.
42
+
43
+ Returns:
44
+ A list of requirements.
45
+ """
46
+ from zenml.integrations.numpy import NumpyIntegration
47
+ from zenml.integrations.pandas import PandasIntegration
48
+
49
+ return cls.REQUIREMENTS + \
50
+ NumpyIntegration.get_requirements(target_os=target_os) + \
51
+ PandasIntegration.get_requirements(target_os=target_os)
52
+
35
53
  @classmethod
36
54
  def flavors(cls) -> List[Type[Flavor]]:
37
55
  """Declare the stack component flavors for the Databricks integration.
@@ -102,6 +102,15 @@ class DatabricksOrchestratorConfig(
102
102
  """
103
103
  return True
104
104
 
105
+ @property
106
+ def is_schedulable(self) -> bool:
107
+ """Whether the orchestrator is schedulable or not.
108
+
109
+ Returns:
110
+ Whether the orchestrator is schedulable or not.
111
+ """
112
+ return True
113
+
105
114
 
106
115
  class DatabricksOrchestratorFlavor(BaseOrchestratorFlavor):
107
116
  """Databricks orchestrator flavor."""
@@ -13,18 +13,17 @@
13
13
  # permissions and limitations under the License.
14
14
  """Deepchecks integration for ZenML.
15
15
 
16
- The Deepchecks integration provides a way to validate your data in your pipelines.
17
- It includes a way to detect data anomalies and define checks to ensure quality of
18
- data.
16
+ The Deepchecks integration provides a way to validate your data in your
17
+ pipelines. It includes a way to detect data anomalies and define checks to
18
+ ensure quality of data.
19
19
 
20
20
  The integration includes custom materializers to store and visualize Deepchecks
21
21
  `SuiteResults`.
22
22
  """
23
23
 
24
- from typing import List, Type
24
+ from typing import List, Type, Optional
25
25
 
26
- from zenml.enums import StackComponentType
27
- from zenml.integrations.constants import DEEPCHECKS
26
+ from zenml.integrations.constants import DEEPCHECKS, PANDAS
28
27
  from zenml.integrations.integration import Integration
29
28
  from zenml.stack import Flavor
30
29
 
@@ -36,21 +35,40 @@ class DeepchecksIntegration(Integration):
36
35
 
37
36
  NAME = DEEPCHECKS
38
37
  REQUIREMENTS = [
39
- "deepchecks[vision]==0.8.0",
38
+ "deepchecks[vision]>=0.18.0",
40
39
  "torchvision>=0.14.0",
41
- "pandas<2.0.0",
42
40
  "opencv-python==4.5.5.64", # pin to same version
43
41
  "opencv-python-headless==4.5.5.64", # pin to same version
44
42
  "tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
43
+ # The deepchecks integrations requires pandas to work.
44
+ # However, their version 0.18.0 is still not compatible with
45
+ # pandas>=2.2.0, so we limit the version here.
46
+ "pandas<2.2.0",
45
47
  ]
48
+
46
49
  APT_PACKAGES = ["ffmpeg", "libsm6", "libxext6"]
47
- REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas","torchvision","tenacity"]
50
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas", "torchvision", "tenacity"]
48
51
 
49
- @staticmethod
50
- def activate() -> None:
52
+ @classmethod
53
+ def activate(cls) -> None:
51
54
  """Activate the Deepchecks integration."""
52
55
  from zenml.integrations.deepchecks import materializers # noqa
53
56
 
57
+ @classmethod
58
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
59
+ """Method to get the requirements for the integration.
60
+
61
+ Args:
62
+ target_os: The target operating system to get the requirements for.
63
+
64
+ Returns:
65
+ A list of requirements.
66
+ """
67
+ from zenml.integrations.pandas import PandasIntegration
68
+
69
+ return cls.REQUIREMENTS + \
70
+ PandasIntegration.get_requirements(target_os=target_os)
71
+
54
72
  @classmethod
55
73
  def flavors(cls) -> List[Type[Flavor]]:
56
74
  """Declare the stack component flavors for the Deepchecks integration.
@@ -18,7 +18,9 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, Tuple, Type
18
18
  from deepchecks.tabular import Dataset
19
19
 
20
20
  from zenml.enums import ArtifactType, VisualizationType
21
- from zenml.materializers.pandas_materializer import PandasMaterializer
21
+ from zenml.integrations.pandas.materializers.pandas_materializer import (
22
+ PandasMaterializer,
23
+ )
22
24
 
23
25
  if TYPE_CHECKING:
24
26
  from zenml.metadata.metadata_types import MetadataType
@@ -176,9 +176,6 @@ class DeepchecksDataIntegrityCheck(DeepchecksValidationCheck):
176
176
  TABULAR_FEATURE_LABEL_CORRELATION = source_utils.resolve(
177
177
  tabular_checks.FeatureLabelCorrelation
178
178
  ).import_path
179
- TABULAR_IDENTIFIER_LEAKAGE = source_utils.resolve(
180
- tabular_checks.IdentifierLeakage
181
- ).import_path
182
179
  TABULAR_IS_SINGLE_VALUE = source_utils.resolve(
183
180
  tabular_checks.IsSingleValue
184
181
  ).import_path
@@ -234,9 +231,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
234
231
  TABULAR_DATE_TRAIN_TEST_LEAKAGE_OVERLAP = source_utils.resolve(
235
232
  tabular_checks.DateTrainTestLeakageOverlap
236
233
  ).import_path
237
- TABULAR_DOMINANT_FREQUENCY_CHANGE = source_utils.resolve(
238
- tabular_checks.DominantFrequencyChange
239
- ).import_path
240
234
  TABULAR_FEATURE_LABEL_CORRELATION_CHANGE = source_utils.resolve(
241
235
  tabular_checks.FeatureLabelCorrelationChange
242
236
  ).import_path
@@ -262,9 +256,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
262
256
  tabular_checks.WholeDatasetDrift
263
257
  ).import_path
264
258
 
265
- VISION_FEATURE_LABEL_CORRELATION_CHANGE = source_utils.resolve(
266
- vision_checks.FeatureLabelCorrelationChange
267
- ).import_path
268
259
  VISION_HEATMAP_COMPARISON = source_utils.resolve(
269
260
  vision_checks.HeatmapComparison
270
261
  ).import_path
@@ -277,9 +268,6 @@ class DeepchecksDataDriftCheck(DeepchecksValidationCheck):
277
268
  VISION_NEW_LABELS = source_utils.resolve(
278
269
  vision_checks.NewLabels
279
270
  ).import_path
280
- VISION_SIMILAR_IMAGE_LEAKAGE = source_utils.resolve(
281
- vision_checks.SimilarImageLeakage
282
- ).import_path
283
271
  VISION_TRAIN_TEST_LABEL_DRIFT = source_utils.resolve(
284
272
  vision_checks.TrainTestLabelDrift
285
273
  ).import_path
@@ -324,21 +312,12 @@ class DeepchecksModelValidationCheck(DeepchecksValidationCheck):
324
312
  VISION_CONFUSION_MATRIX_REPORT = source_utils.resolve(
325
313
  vision_checks.ConfusionMatrixReport
326
314
  ).import_path
327
- VISION_IMAGE_SEGMENT_PERFORMANCE = source_utils.resolve(
328
- vision_checks.ImageSegmentPerformance
329
- ).import_path
330
315
  VISION_MEAN_AVERAGE_PRECISION_REPORT = source_utils.resolve(
331
316
  vision_checks.MeanAveragePrecisionReport
332
317
  ).import_path
333
318
  VISION_MEAN_AVERAGE_RECALL_REPORT = source_utils.resolve(
334
319
  vision_checks.MeanAverageRecallReport
335
320
  ).import_path
336
- VISION_ROBUSTNESS_REPORT = source_utils.resolve(
337
- vision_checks.RobustnessReport
338
- ).import_path
339
- VISION_SINGLE_DATASET_SCALAR_PERFORMANCE = source_utils.resolve(
340
- vision_checks.SingleDatasetScalarPerformance
341
- ).import_path
342
321
 
343
322
 
344
323
  class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
@@ -358,12 +337,6 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
358
337
  TABULAR_BOOSTING_OVERFIT = source_utils.resolve(
359
338
  tabular_checks.BoostingOverfit
360
339
  ).import_path
361
- TABULAR_MODEL_ERROR_ANALYSIS = source_utils.resolve(
362
- tabular_checks.ModelErrorAnalysis
363
- ).import_path
364
- TABULAR_PERFORMANCE_REPORT = source_utils.resolve(
365
- tabular_checks.PerformanceReport
366
- ).import_path
367
340
  TABULAR_SIMPLE_MODEL_COMPARISON = source_utils.resolve(
368
341
  tabular_checks.SimpleModelComparison
369
342
  ).import_path
@@ -377,9 +350,6 @@ class DeepchecksModelDriftCheck(DeepchecksValidationCheck):
377
350
  VISION_CLASS_PERFORMANCE = source_utils.resolve(
378
351
  vision_checks.ClassPerformance
379
352
  ).import_path
380
- VISION_MODEL_ERROR_ANALYSIS = source_utils.resolve(
381
- vision_checks.ModelErrorAnalysis
382
- ).import_path
383
353
  VISION_SIMPLE_MODEL_COMPARISON = source_utils.resolve(
384
354
  vision_checks.SimpleModelComparison
385
355
  ).import_path
@@ -25,7 +25,7 @@ file.
25
25
  import logging
26
26
  import os
27
27
  import warnings
28
- from typing import List, Type
28
+ from typing import List, Type, Optional
29
29
 
30
30
  from zenml.integrations.constants import EVIDENTLY
31
31
  from zenml.integrations.integration import Integration
@@ -57,7 +57,22 @@ class EvidentlyIntegration(Integration):
57
57
  "evidently>=0.4.16,<=0.4.22",
58
58
  "tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
59
59
  ]
60
- REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]
60
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity", "pandas"]
61
+
62
+ @classmethod
63
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
64
+ """Method to get the requirements for the integration.
65
+
66
+ Args:
67
+ target_os: The target operating system to get the requirements for.
68
+
69
+ Returns:
70
+ A list of requirements.
71
+ """
72
+ from zenml.integrations.pandas import PandasIntegration
73
+
74
+ return cls.REQUIREMENTS + \
75
+ PandasIntegration.get_requirements(target_os=target_os)
61
76
 
62
77
  @classmethod
63
78
  def flavors(cls) -> List[Type[Flavor]]:
@@ -12,8 +12,8 @@
12
12
  # or implied. See the License for the specific language governing
13
13
  # permissions and limitations under the License.
14
14
  """Facets integration for ZenML."""
15
-
16
- from zenml.integrations.constants import FACETS
15
+ from typing import Optional, List
16
+ from zenml.integrations.constants import FACETS, PANDAS
17
17
  from zenml.integrations.integration import Integration
18
18
 
19
19
 
@@ -21,13 +21,29 @@ class FacetsIntegration(Integration):
21
21
  """Definition of Facets integration for ZenML."""
22
22
 
23
23
  NAME = FACETS
24
- REQUIREMENTS = ["facets-overview>=1.0.0", "pandas"]
24
+ REQUIREMENTS = ["facets-overview>=1.0.0"]
25
+
25
26
  REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
26
27
 
27
- @staticmethod
28
- def activate() -> None:
28
+ @classmethod
29
+ def activate(cls) -> None:
29
30
  """Activate the Facets integration."""
30
31
  from zenml.integrations.facets import materializers # noqa
31
32
 
33
+ @classmethod
34
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
35
+ """Method to get the requirements for the integration.
36
+
37
+ Args:
38
+ target_os: The target operating system to get the requirements for.
39
+
40
+ Returns:
41
+ A list of requirements.
42
+ """
43
+ from zenml.integrations.pandas import PandasIntegration
44
+
45
+ return cls.REQUIREMENTS + \
46
+ PandasIntegration.get_requirements(target_os=target_os)
47
+
32
48
 
33
49
  FacetsIntegration.check_installation()
@@ -17,9 +17,8 @@ The Feast integration offers a way to connect to a Feast Feature Store. ZenML
17
17
  implements a dedicated stack component that you can access as part of your ZenML
18
18
  steps in the usual ways.
19
19
  """
20
- from typing import List, Type
20
+ from typing import List, Type, Optional
21
21
 
22
- from zenml.enums import StackComponentType
23
22
  from zenml.integrations.constants import FEAST
24
23
  from zenml.integrations.integration import Integration
25
24
  from zenml.stack import Flavor
@@ -31,10 +30,9 @@ class FeastIntegration(Integration):
31
30
  """Definition of Feast integration for ZenML."""
32
31
 
33
32
  NAME = FEAST
34
- # click is added to keep the feast click version
35
- # in sync with ZenML's click
36
- REQUIREMENTS = ["feast<=0.37.1", "click>=8.0.1,<8.1.4"]
37
- REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click"]
33
+ # click is added to keep the feast click version in sync with ZenML's click
34
+ REQUIREMENTS = ["feast", "click>=8.0.1,<8.1.4"]
35
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click", "pandas"]
38
36
 
39
37
  @classmethod
40
38
  def flavors(cls) -> List[Type[Flavor]]:
@@ -47,5 +45,20 @@ class FeastIntegration(Integration):
47
45
 
48
46
  return [FeastFeatureStoreFlavor]
49
47
 
48
+ @classmethod
49
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
50
+ """Method to get the requirements for the integration.
51
+
52
+ Args:
53
+ target_os: The target operating system to get the requirements for.
54
+
55
+ Returns:
56
+ A list of requirements.
57
+ """
58
+ from zenml.integrations.pandas import PandasIntegration
59
+
60
+ return cls.REQUIREMENTS + \
61
+ PandasIntegration.get_requirements(target_os=target_os)
62
+
50
63
 
51
64
  FeastIntegration.check_installation()
@@ -55,8 +55,8 @@ class GcpIntegration(Integration):
55
55
  ]
56
56
  REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes","kfp"]
57
57
 
58
- @staticmethod
59
- def activate() -> None:
58
+ @classmethod
59
+ def activate(cls) -> None:
60
60
  """Activate the GCP integration."""
61
61
  from zenml.integrations.gcp import service_connectors # noqa
62
62
 
@@ -162,6 +162,15 @@ class VertexOrchestratorConfig(
162
162
  """
163
163
  return self.synchronous
164
164
 
165
+ @property
166
+ def is_schedulable(self) -> bool:
167
+ """Whether the orchestrator is schedulable or not.
168
+
169
+ Returns:
170
+ Whether the orchestrator is schedulable or not.
171
+ """
172
+ return True
173
+
165
174
 
166
175
  class VertexOrchestratorFlavor(BaseOrchestratorFlavor):
167
176
  """Vertex Orchestrator flavor."""
@@ -32,7 +32,16 @@
32
32
  import os
33
33
  import re
34
34
  import types
35
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, cast
35
+ from typing import (
36
+ TYPE_CHECKING,
37
+ Any,
38
+ Dict,
39
+ List,
40
+ Optional,
41
+ Tuple,
42
+ Type,
43
+ cast,
44
+ )
36
45
  from uuid import UUID
37
46
 
38
47
  from google.api_core import exceptions as google_exceptions
@@ -17,9 +17,8 @@ The Great Expectations integration enables you to use Great Expectations as a
17
17
  way of profiling and validating your data.
18
18
  """
19
19
 
20
- from typing import List, Type
20
+ from typing import List, Type, Optional
21
21
 
22
- from zenml.enums import StackComponentType
23
22
  from zenml.integrations.constants import GREAT_EXPECTATIONS
24
23
  from zenml.integrations.integration import Integration
25
24
  from zenml.stack import Flavor
@@ -31,12 +30,12 @@ class GreatExpectationsIntegration(Integration):
31
30
  """Definition of Great Expectations integration for ZenML."""
32
31
 
33
32
  NAME = GREAT_EXPECTATIONS
34
- REQUIREMENTS = [
35
- "great-expectations>=0.17.15,<1.0",
36
- ]
33
+ REQUIREMENTS = ["great-expectations>=0.17.15,<1.0"]
37
34
 
38
- @staticmethod
39
- def activate() -> None:
35
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
36
+
37
+ @classmethod
38
+ def activate(cls) -> None:
40
39
  """Activate the Great Expectations integration."""
41
40
  from zenml.integrations.great_expectations import materializers # noqa
42
41
 
@@ -53,5 +52,20 @@ class GreatExpectationsIntegration(Integration):
53
52
 
54
53
  return [GreatExpectationsDataValidatorFlavor]
55
54
 
55
+ @classmethod
56
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
57
+ """Method to get the requirements for the integration.
58
+
59
+ Args:
60
+ target_os: The target operating system to get the requirements for.
61
+
62
+ Returns:
63
+ A list of requirements.
64
+ """
65
+ from zenml.integrations.pandas import PandasIntegration
66
+
67
+ return cls.REQUIREMENTS + \
68
+ PandasIntegration.get_requirements(target_os=target_os)
69
+
56
70
 
57
71
  GreatExpectationsIntegration.check_installation()
@@ -12,7 +12,8 @@
12
12
  # or implied. See the License for the specific language governing
13
13
  # permissions and limitations under the License.
14
14
  """Initialization of the Huggingface integration."""
15
- from typing import List, Type
15
+ import sys
16
+ from typing import List, Type, Optional
16
17
 
17
18
  from zenml.integrations.constants import HUGGINGFACE
18
19
  from zenml.integrations.integration import Integration
@@ -26,20 +27,8 @@ class HuggingfaceIntegration(Integration):
26
27
  """Definition of Huggingface integration for ZenML."""
27
28
 
28
29
  NAME = HUGGINGFACE
29
- REQUIREMENTS = [
30
- "transformers<=4.31",
31
- "datasets",
32
- "huggingface_hub>0.19.0",
33
- "accelerate",
34
- "bitsandbytes>=0.41.3",
35
- "peft",
36
- # temporary fix for CI issue similar to:
37
- # - https://github.com/huggingface/datasets/issues/6737
38
- # - https://github.com/huggingface/datasets/issues/6697
39
- # TODO try relaxing it back going forward
40
- "fsspec<=2023.12.0",
41
- ]
42
- REQUIREMENTS_IGNORED_ON_UNINSTALL = ["fsspec"]
30
+
31
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["fsspec", "pandas"]
43
32
 
44
33
  @classmethod
45
34
  def activate(cls) -> None:
@@ -47,6 +36,41 @@ class HuggingfaceIntegration(Integration):
47
36
  from zenml.integrations.huggingface import materializers # noqa
48
37
  from zenml.integrations.huggingface import services
49
38
 
39
+ @classmethod
40
+ def get_requirements(cls, target_os: Optional[str] = None) -> List[str]:
41
+ """Defines platform specific requirements for the integration.
42
+
43
+ Args:
44
+ target_os: The target operating system.
45
+
46
+ Returns:
47
+ A list of requirements.
48
+ """
49
+ requirements = [
50
+ "datasets",
51
+ "huggingface_hub>0.19.0",
52
+ "accelerate",
53
+ "bitsandbytes>=0.41.3",
54
+ "peft",
55
+ # temporary fix for CI issue similar to:
56
+ # - https://github.com/huggingface/datasets/issues/6737
57
+ # - https://github.com/huggingface/datasets/issues/6697
58
+ # TODO try relaxing it back going forward
59
+ "fsspec<=2023.12.0",
60
+ ]
61
+
62
+ # In python 3.8 higher transformers version lead to other packages breaking
63
+ if sys.version_info.minor > 8:
64
+ requirements += ["transformers"]
65
+ else:
66
+ requirements += ["transformers<=4.31"]
67
+
68
+ # Add the pandas integration requirements
69
+ from zenml.integrations.pandas import PandasIntegration
70
+
71
+ return requirements + \
72
+ PandasIntegration.get_requirements(target_os=target_os)
73
+
50
74
  @classmethod
51
75
  def flavors(cls) -> List[Type[Flavor]]:
52
76
  """Declare the stack component flavors for the Huggingface integration.
@@ -25,3 +25,6 @@ from zenml.integrations.huggingface.materializers.huggingface_tf_model_materiali
25
25
  from zenml.integrations.huggingface.materializers.huggingface_tokenizer_materializer import (
26
26
  HFTokenizerMaterializer,
27
27
  )
28
+ from zenml.integrations.huggingface.materializers.huggingface_t5_materializer import (
29
+ HFT5Materializer,
30
+ )
@@ -31,9 +31,11 @@ from datasets import Dataset, load_from_disk
31
31
  from datasets.dataset_dict import DatasetDict
32
32
 
33
33
  from zenml.enums import ArtifactType, VisualizationType
34
+ from zenml.integrations.pandas.materializers.pandas_materializer import (
35
+ PandasMaterializer,
36
+ )
34
37
  from zenml.io import fileio
35
38
  from zenml.materializers.base_materializer import BaseMaterializer
36
- from zenml.materializers.pandas_materializer import PandasMaterializer
37
39
  from zenml.utils import io_utils
38
40
 
39
41
  if TYPE_CHECKING:
@@ -18,7 +18,7 @@ import os
18
18
  from tempfile import TemporaryDirectory
19
19
  from typing import Any, ClassVar, Dict, Tuple, Type
20
20
 
21
- from transformers import ( # type: ignore [import-untyped]
21
+ from transformers import (
22
22
  AutoConfig,
23
23
  PreTrainedModel,
24
24
  )
@@ -0,0 +1,107 @@
1
+ # Copyright (c) ZenML GmbH 2024. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at:
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
+ # or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ """Implementation of the Huggingface t5 materializer."""
15
+
16
+ import os
17
+ import tempfile
18
+ from typing import Any, ClassVar, Type, Union
19
+
20
+ from transformers import (
21
+ T5ForConditionalGeneration,
22
+ T5Tokenizer,
23
+ T5TokenizerFast,
24
+ )
25
+
26
+ from zenml.io import fileio
27
+ from zenml.materializers.base_materializer import BaseMaterializer
28
+
29
+
30
+ class HFT5Materializer(BaseMaterializer):
31
+ """Base class for huggingface t5 models."""
32
+
33
+ SKIP_REGISTRATION: ClassVar[bool] = False
34
+ ASSOCIATED_TYPES = (
35
+ T5ForConditionalGeneration,
36
+ T5Tokenizer,
37
+ T5TokenizerFast,
38
+ )
39
+
40
+ def load(
41
+ self, data_type: Type[Any]
42
+ ) -> Union[T5ForConditionalGeneration, T5Tokenizer, T5TokenizerFast]:
43
+ """Reads a T5ForConditionalGeneration model or T5Tokenizer from a serialized zip file.
44
+
45
+ Args:
46
+ data_type: A T5ForConditionalGeneration or T5Tokenizer type.
47
+
48
+ Returns:
49
+ A T5ForConditionalGeneration or T5Tokenizer object.
50
+
51
+ Raises:
52
+ ValueError: Unsupported data type used
53
+ """
54
+ filepath = self.uri
55
+
56
+ with tempfile.TemporaryDirectory(prefix="zenml-temp-") as temp_dir:
57
+ # Copy files from artifact store to temporary directory
58
+ for file in fileio.listdir(filepath):
59
+ src = os.path.join(filepath, file)
60
+ dst = os.path.join(temp_dir, file)
61
+ if fileio.isdir(src):
62
+ fileio.makedirs(dst)
63
+ for subfile in fileio.listdir(src):
64
+ subsrc = os.path.join(src, subfile)
65
+ subdst = os.path.join(dst, subfile)
66
+ fileio.copy(subsrc, subdst)
67
+ else:
68
+ fileio.copy(src, dst)
69
+
70
+ # Load the model or tokenizer from the temporary directory
71
+ if data_type in [
72
+ T5ForConditionalGeneration,
73
+ T5Tokenizer,
74
+ T5TokenizerFast,
75
+ ]:
76
+ return data_type.from_pretrained(temp_dir)
77
+ else:
78
+ raise ValueError(f"Unsupported data type: {data_type}")
79
+
80
+ def save(
81
+ self,
82
+ obj: Union[T5ForConditionalGeneration, T5Tokenizer, T5TokenizerFast],
83
+ ) -> None:
84
+ """Creates a serialization for a T5ForConditionalGeneration model or T5Tokenizer.
85
+
86
+ Args:
87
+ obj: A T5ForConditionalGeneration model or T5Tokenizer.
88
+ """
89
+ # Create a temporary directory
90
+ with tempfile.TemporaryDirectory(prefix="zenml-temp-") as temp_dir:
91
+ # Save the model or tokenizer
92
+ obj.save_pretrained(temp_dir)
93
+
94
+ # Copy the directory to the artifact store
95
+ filepath = self.uri
96
+ fileio.makedirs(filepath)
97
+ for file in os.listdir(temp_dir):
98
+ src = os.path.join(temp_dir, file)
99
+ dst = os.path.join(filepath, file)
100
+ if os.path.isdir(src):
101
+ fileio.makedirs(dst)
102
+ for subfile in os.listdir(src):
103
+ subsrc = os.path.join(src, subfile)
104
+ subdst = os.path.join(dst, subfile)
105
+ fileio.copy(subsrc, subdst)
106
+ else:
107
+ fileio.copy(src, dst)
@@ -18,7 +18,7 @@ import os
18
18
  from tempfile import TemporaryDirectory
19
19
  from typing import Any, ClassVar, Dict, Tuple, Type
20
20
 
21
- from transformers import ( # type: ignore [import-untyped]
21
+ from transformers import (
22
22
  AutoConfig,
23
23
  TFPreTrainedModel,
24
24
  )