zenml-nightly 0.62.0.dev20240729__py3-none-any.whl → 0.64.0.dev20240809__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. README.md +2 -2
  2. RELEASE_NOTES.md +120 -0
  3. zenml/VERSION +1 -1
  4. zenml/__init__.py +0 -4
  5. zenml/actions/pipeline_run/pipeline_run_action.py +19 -17
  6. zenml/analytics/enums.py +4 -6
  7. zenml/cli/__init__.py +28 -76
  8. zenml/cli/base.py +2 -2
  9. zenml/cli/pipeline.py +54 -61
  10. zenml/cli/stack.py +6 -8
  11. zenml/cli/web_login.py +8 -0
  12. zenml/client.py +232 -103
  13. zenml/config/build_configuration.py +43 -17
  14. zenml/config/compiler.py +14 -22
  15. zenml/config/docker_settings.py +80 -57
  16. zenml/config/pipeline_run_configuration.py +3 -0
  17. zenml/config/server_config.py +3 -0
  18. zenml/config/source.py +60 -1
  19. zenml/constants.py +11 -2
  20. zenml/entrypoints/base_entrypoint_configuration.py +53 -8
  21. zenml/enums.py +4 -1
  22. zenml/environment.py +25 -9
  23. zenml/image_builders/base_image_builder.py +1 -1
  24. zenml/image_builders/build_context.py +25 -72
  25. zenml/integrations/aws/orchestrators/sagemaker_orchestrator.py +13 -4
  26. zenml/integrations/azure/__init__.py +4 -0
  27. zenml/integrations/azure/flavors/__init__.py +11 -0
  28. zenml/integrations/azure/flavors/azureml_orchestrator_flavor.py +263 -0
  29. zenml/{_hub → integrations/azure/orchestrators}/__init__.py +7 -2
  30. zenml/integrations/azure/orchestrators/azureml_orchestrator.py +544 -0
  31. zenml/integrations/azure/orchestrators/azureml_orchestrator_entrypoint_config.py +86 -0
  32. zenml/integrations/azure/step_operators/azureml_step_operator.py +3 -0
  33. zenml/integrations/databricks/flavors/databricks_orchestrator_flavor.py +20 -2
  34. zenml/integrations/databricks/orchestrators/databricks_orchestrator.py +19 -13
  35. zenml/integrations/gcp/orchestrators/vertex_orchestrator.py +7 -2
  36. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +123 -6
  37. zenml/integrations/kaniko/image_builders/kaniko_image_builder.py +1 -1
  38. zenml/integrations/mlflow/__init__.py +1 -1
  39. zenml/integrations/mlflow/experiment_trackers/mlflow_experiment_tracker.py +3 -1
  40. zenml/integrations/mlflow/flavors/mlflow_experiment_tracker_flavor.py +3 -0
  41. zenml/logger.py +13 -0
  42. zenml/models/__init__.py +26 -22
  43. zenml/models/v2/base/filter.py +32 -0
  44. zenml/models/v2/core/pipeline.py +73 -89
  45. zenml/models/v2/core/pipeline_build.py +15 -11
  46. zenml/models/v2/core/pipeline_deployment.py +72 -24
  47. zenml/models/v2/core/pipeline_run.py +65 -1
  48. zenml/models/v2/core/run_template.py +393 -0
  49. zenml/models/v2/core/server_settings.py +12 -0
  50. zenml/models/v2/core/user.py +0 -21
  51. zenml/models/v2/misc/server_models.py +7 -1
  52. zenml/models/v2/misc/stack_deployment.py +5 -0
  53. zenml/models/v2/misc/user_auth.py +0 -7
  54. zenml/new/pipelines/build_utils.py +220 -89
  55. zenml/new/pipelines/code_archive.py +157 -0
  56. zenml/new/pipelines/pipeline.py +46 -78
  57. zenml/new/pipelines/run_utils.py +79 -1
  58. zenml/post_execution/pipeline.py +1 -4
  59. zenml/service_connectors/service_connector_utils.py +18 -2
  60. zenml/stack_deployments/aws_stack_deployment.py +32 -8
  61. zenml/stack_deployments/azure_stack_deployment.py +122 -10
  62. zenml/stack_deployments/gcp_stack_deployment.py +36 -7
  63. zenml/stack_deployments/stack_deployment.py +23 -7
  64. zenml/steps/base_step.py +3 -0
  65. zenml/steps/utils.py +0 -4
  66. zenml/utils/archivable.py +149 -0
  67. zenml/utils/code_utils.py +244 -0
  68. zenml/utils/notebook_utils.py +122 -0
  69. zenml/utils/package_utils.py +39 -0
  70. zenml/utils/pipeline_docker_image_builder.py +3 -96
  71. zenml/utils/source_utils.py +109 -1
  72. zenml/zen_server/dashboard/assets/{404-B_YdvmwS.js → 404-CRAA_Lew.js} +1 -1
  73. zenml/zen_server/dashboard/assets/@radix-BXWm7HOa.js +85 -0
  74. zenml/zen_server/dashboard/assets/{@react-router-CO-OsFwI.js → @react-router-l3lMcXA2.js} +1 -1
  75. zenml/zen_server/dashboard/assets/{@reactflow-l_1hUr1S.js → @reactflow-CeVxyqYT.js} +2 -2
  76. zenml/zen_server/dashboard/assets/{@tanstack-DYiOyJUL.js → @tanstack-FmcYZMuX.js} +4 -4
  77. zenml/zen_server/dashboard/assets/AlertDialogDropdownItem-ErO9aOgK.js +1 -0
  78. zenml/zen_server/dashboard/assets/{AwarenessChannel-CFg5iX4Z.js → AwarenessChannel-CLXo5rKM.js} +1 -1
  79. zenml/zen_server/dashboard/assets/{CodeSnippet-Dvkx_82E.js → CodeSnippet-D0VLxT2A.js} +2 -2
  80. zenml/zen_server/dashboard/assets/CollapsibleCard-BaUPiVg0.js +1 -0
  81. zenml/zen_server/dashboard/assets/{Commands-DoN1xrEq.js → Commands-JrcZK-3j.js} +1 -1
  82. zenml/zen_server/dashboard/assets/CopyButton-Dbo52T1K.js +2 -0
  83. zenml/zen_server/dashboard/assets/{CsvVizualization-Ck-nZ43m.js → CsvVizualization-D3kAypDj.js} +3 -3
  84. zenml/zen_server/dashboard/assets/DisplayDate-DizbSeT-.js +1 -0
  85. zenml/zen_server/dashboard/assets/EditSecretDialog-Bd7mFLS4.js +1 -0
  86. zenml/zen_server/dashboard/assets/{EmptyState-BMLnFVlB.js → EmptyState-BHblM39I.js} +1 -1
  87. zenml/zen_server/dashboard/assets/{Error-kLtljEOM.js → Error-C6LeJSER.js} +1 -1
  88. zenml/zen_server/dashboard/assets/{ExecutionStatus-DguLLgTK.js → ExecutionStatus-jH4OrWBq.js} +1 -1
  89. zenml/zen_server/dashboard/assets/{Helpbox-BXUMP21n.js → Helpbox-aAB2XP-z.js} +1 -1
  90. zenml/zen_server/dashboard/assets/{Infobox-DSt0O-dm.js → Infobox-BQ0aty32.js} +1 -1
  91. zenml/zen_server/dashboard/assets/{InlineAvatar-xsrsIGE-.js → InlineAvatar-DpTLgM3Q.js} +1 -1
  92. zenml/zen_server/dashboard/assets/Lock-CNyJvf2r.js +1 -0
  93. zenml/zen_server/dashboard/assets/{MarkdownVisualization-xp3hhULl.js → MarkdownVisualization-Bajxn0HY.js} +1 -1
  94. zenml/zen_server/dashboard/assets/NumberBox-BmKE0qnO.js +1 -0
  95. zenml/zen_server/dashboard/assets/{PasswordChecker-DUveqlva.js → PasswordChecker-yGGoJSB-.js} +1 -1
  96. zenml/zen_server/dashboard/assets/ProviderRadio-BBqkIuTd.js +1 -0
  97. zenml/zen_server/dashboard/assets/RadioItem-xLhXoiFV.js +1 -0
  98. zenml/zen_server/dashboard/assets/SearchField-C9R0mdaX.js +1 -0
  99. zenml/zen_server/dashboard/assets/{SetPassword-BXGTWiwj.js → SetPassword-52sNxNiO.js} +1 -1
  100. zenml/zen_server/dashboard/assets/{SuccessStep-DZC60t0x.js → SuccessStep-DlkItqYG.js} +1 -1
  101. zenml/zen_server/dashboard/assets/Tick-uxv80Q6a.js +1 -0
  102. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-DGvwFWO1.js → UpdatePasswordSchemas-oN4G3sKz.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{aws-BgKTfTfx.js → aws-0_3UsPif.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{check-circle-i56092KI.js → check-circle-1_I207rW.js} +1 -1
  105. zenml/zen_server/dashboard/assets/chevron-down-BpaF8JqM.js +1 -0
  106. zenml/zen_server/dashboard/assets/{chevron-right-double-CZBOf6JM.js → chevron-right-double-Dk8e2L99.js} +1 -1
  107. zenml/zen_server/dashboard/assets/{cloud-only-C_yFCAkP.js → cloud-only-BkUuI0lZ.js} +1 -1
  108. zenml/zen_server/dashboard/assets/components-Br2ezRib.js +1 -0
  109. zenml/zen_server/dashboard/assets/{copy-BXNk6BjL.js → copy-f3XGPPxt.js} +1 -1
  110. zenml/zen_server/dashboard/assets/{database-1xWSgZfO.js → database-cXYNX9tt.js} +1 -1
  111. zenml/zen_server/dashboard/assets/{docker-CQMVm_4d.js → docker-8uj__HHK.js} +1 -1
  112. zenml/zen_server/dashboard/assets/dots-horizontal-sKQlWEni.js +1 -0
  113. zenml/zen_server/dashboard/assets/edit-C0MVvPD2.js +1 -0
  114. zenml/zen_server/dashboard/assets/{file-text-CqD_iu6l.js → file-text-B9JibxTs.js} +1 -1
  115. zenml/zen_server/dashboard/assets/{help-bu_DgLKI.js → help-FuHlZwn0.js} +1 -1
  116. zenml/zen_server/dashboard/assets/{index-rK_Wuy2W.js → index-Bd1xgUQG.js} +1 -1
  117. zenml/zen_server/dashboard/assets/index-DaGknux4.css +1 -0
  118. zenml/zen_server/dashboard/assets/{index-BczVOqUf.js → index-DhIZtpxB.js} +5 -5
  119. zenml/zen_server/dashboard/assets/index.esm-DT4uyn2i.js +1 -0
  120. zenml/zen_server/dashboard/assets/layout-D6oiSbfd.js +1 -0
  121. zenml/zen_server/dashboard/assets/{login-mutation-CrHrndTI.js → login-mutation-13A_JSVA.js} +1 -1
  122. zenml/zen_server/dashboard/assets/{logs-D8k8BVFf.js → logs-CgeE2vZP.js} +1 -1
  123. zenml/zen_server/dashboard/assets/{not-found-DYa4pC-C.js → not-found-B0Mmb90p.js} +1 -1
  124. zenml/zen_server/dashboard/assets/package-DdkziX79.js +1 -0
  125. zenml/zen_server/dashboard/assets/page-7-v2OBm-.js +1 -0
  126. zenml/zen_server/dashboard/assets/{page-MFQyIJd3.js → page-B3ozwdD1.js} +1 -1
  127. zenml/zen_server/dashboard/assets/{page-BkuQDIf-.js → page-BGwA9B1M.js} +1 -1
  128. zenml/zen_server/dashboard/assets/{page-1iL8aMqs.js → page-BkjAUyTA.js} +1 -1
  129. zenml/zen_server/dashboard/assets/page-BnacgBiy.js +1 -0
  130. zenml/zen_server/dashboard/assets/page-BxF_KMQ3.js +2 -0
  131. zenml/zen_server/dashboard/assets/page-C4POHC0K.js +1 -0
  132. zenml/zen_server/dashboard/assets/page-C9kudd44.js +9 -0
  133. zenml/zen_server/dashboard/assets/page-CA1j3GpJ.js +1 -0
  134. zenml/zen_server/dashboard/assets/page-CCY6yfmu.js +1 -0
  135. zenml/zen_server/dashboard/assets/page-CgTe7Bme.js +1 -0
  136. zenml/zen_server/dashboard/assets/{page-8a4UMKXZ.js → page-Cgn-6v2Y.js} +1 -1
  137. zenml/zen_server/dashboard/assets/page-CxQmQqDw.js +1 -0
  138. zenml/zen_server/dashboard/assets/page-D2Goey3H.js +1 -0
  139. zenml/zen_server/dashboard/assets/page-DLpOnf7u.js +1 -0
  140. zenml/zen_server/dashboard/assets/{page-BhgCDInH.js → page-DSTQnBk-.js} +1 -1
  141. zenml/zen_server/dashboard/assets/{page-1h_sD1jz.js → page-DTysUGOy.js} +1 -1
  142. zenml/zen_server/dashboard/assets/{page-2grKx_MY.js → page-D_EXUFJb.js} +1 -1
  143. zenml/zen_server/dashboard/assets/page-Db15QzsM.js +1 -0
  144. zenml/zen_server/dashboard/assets/{page-BDns21Iz.js → page-DugsjcQ_.js} +1 -1
  145. zenml/zen_server/dashboard/assets/{page-C6-UGEbH.js → page-OFKSPyN7.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{page-BkeAAYwp.js → page-RnG-qhv9.js} +1 -1
  147. zenml/zen_server/dashboard/assets/{page-CCNRIt_f.js → page-T2BtjwPl.js} +1 -1
  148. zenml/zen_server/dashboard/assets/page-TXe1Eo3Z.js +1 -0
  149. zenml/zen_server/dashboard/assets/{page-BnaevhnB.js → page-YiF_fNbe.js} +1 -1
  150. zenml/zen_server/dashboard/assets/{page-uA5prJGY.js → page-hQaiQXfg.js} +1 -1
  151. zenml/zen_server/dashboard/assets/persist-3-5nOJ6m.js +1 -0
  152. zenml/zen_server/dashboard/assets/{play-circle-CNtZKDnW.js → play-circle-XSkLR12B.js} +1 -1
  153. zenml/zen_server/dashboard/assets/plus-FB9-lEq_.js +1 -0
  154. zenml/zen_server/dashboard/assets/refresh-COb6KYDi.js +1 -0
  155. zenml/zen_server/dashboard/assets/sharedSchema-BoYx_B_L.js +14 -0
  156. zenml/zen_server/dashboard/assets/{stack-detail-query-Cficsl6d.js → stack-detail-query-B-US_-wa.js} +1 -1
  157. zenml/zen_server/dashboard/assets/{terminal-By9cErXc.js → terminal-grtjrIEJ.js} +1 -1
  158. zenml/zen_server/dashboard/assets/trash-Cd5CSFqA.js +1 -0
  159. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-7d8xi1tS.js → update-server-settings-mutation-B8GB_ubU.js} +1 -1
  160. zenml/zen_server/dashboard/assets/{url-D7mAQGUM.js → url-hcMJkz8p.js} +1 -1
  161. zenml/zen_server/dashboard/assets/{zod-BhoGpZ63.js → zod-CnykDKJj.js} +1 -1
  162. zenml/zen_server/dashboard/index.html +7 -7
  163. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  164. zenml/zen_server/dashboard_legacy/index.html +1 -1
  165. zenml/zen_server/dashboard_legacy/{precache-manifest.12246c7548e71e2c4438e496360de80c.js → precache-manifest.9c473c96a43298343a7ce1256183123b.js} +4 -4
  166. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  167. zenml/zen_server/dashboard_legacy/static/js/{main.3b27024b.chunk.js → main.463c90b9.chunk.js} +2 -2
  168. zenml/zen_server/dashboard_legacy/static/js/{main.3b27024b.chunk.js.map → main.463c90b9.chunk.js.map} +1 -1
  169. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  170. zenml/zen_server/deploy/helm/README.md +2 -2
  171. zenml/zen_server/rbac/models.py +1 -0
  172. zenml/zen_server/rbac/utils.py +4 -0
  173. zenml/zen_server/routers/pipeline_builds_endpoints.py +2 -66
  174. zenml/zen_server/routers/pipeline_deployments_endpoints.py +2 -53
  175. zenml/zen_server/routers/pipelines_endpoints.py +1 -74
  176. zenml/zen_server/routers/run_templates_endpoints.py +212 -0
  177. zenml/zen_server/routers/stack_deployment_endpoints.py +6 -0
  178. zenml/zen_server/routers/users_endpoints.py +0 -7
  179. zenml/zen_server/routers/workspaces_endpoints.py +79 -0
  180. zenml/zen_server/{pipeline_deployment → template_execution}/runner_entrypoint_configuration.py +1 -8
  181. zenml/zen_server/{pipeline_deployment → template_execution}/utils.py +214 -92
  182. zenml/zen_server/utils.py +77 -2
  183. zenml/zen_server/zen_server_api.py +54 -2
  184. zenml/zen_stores/base_zen_store.py +7 -1
  185. zenml/zen_stores/migrations/versions/0.63.0_release.py +23 -0
  186. zenml/zen_stores/migrations/versions/0.64.0_release.py +23 -0
  187. zenml/zen_stores/migrations/versions/026d4577b6a0_add_code_path.py +39 -0
  188. zenml/zen_stores/migrations/versions/3dcc5d20e82f_add_last_user_activity.py +51 -0
  189. zenml/zen_stores/migrations/versions/7d1919bb1ef0_add_run_templates.py +100 -0
  190. zenml/zen_stores/migrations/versions/909550c7c4da_remove_user_hub_token.py +36 -0
  191. zenml/zen_stores/migrations/versions/b59aa68fdb1f_simplify_pipelines.py +139 -0
  192. zenml/zen_stores/rest_zen_store.py +112 -39
  193. zenml/zen_stores/schemas/__init__.py +2 -0
  194. zenml/zen_stores/schemas/pipeline_build_schemas.py +3 -3
  195. zenml/zen_stores/schemas/pipeline_deployment_schemas.py +32 -2
  196. zenml/zen_stores/schemas/pipeline_run_schemas.py +29 -3
  197. zenml/zen_stores/schemas/pipeline_schemas.py +29 -30
  198. zenml/zen_stores/schemas/run_template_schemas.py +264 -0
  199. zenml/zen_stores/schemas/server_settings_schemas.py +2 -0
  200. zenml/zen_stores/schemas/step_run_schemas.py +11 -4
  201. zenml/zen_stores/schemas/user_schemas.py +0 -2
  202. zenml/zen_stores/sql_zen_store.py +389 -151
  203. zenml/zen_stores/template_utils.py +261 -0
  204. zenml/zen_stores/zen_store_interface.py +93 -20
  205. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/METADATA +3 -3
  206. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/RECORD +211 -184
  207. zenml/_hub/client.py +0 -289
  208. zenml/_hub/constants.py +0 -21
  209. zenml/_hub/utils.py +0 -79
  210. zenml/cli/hub.py +0 -1116
  211. zenml/models/v2/core/pipeline_namespace.py +0 -113
  212. zenml/models/v2/misc/hub_plugin_models.py +0 -79
  213. zenml/new/pipelines/deserialization_utils.py +0 -292
  214. zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +0 -85
  215. zenml/zen_server/dashboard/assets/CollapsibleCard-opiuBHHc.js +0 -1
  216. zenml/zen_server/dashboard/assets/CopyButton-Cr7xYEPb.js +0 -2
  217. zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +0 -1
  218. zenml/zen_server/dashboard/assets/Pagination-C6X-mifw.js +0 -1
  219. zenml/zen_server/dashboard/assets/index-EpMIKgrI.css +0 -1
  220. zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +0 -1
  221. zenml/zen_server/dashboard/assets/package-B3fWP-Dh.js +0 -1
  222. zenml/zen_server/dashboard/assets/page-5NCOHOsy.js +0 -1
  223. zenml/zen_server/dashboard/assets/page-B6h3iaHJ.js +0 -1
  224. zenml/zen_server/dashboard/assets/page-Bi-wtWiO.js +0 -5
  225. zenml/zen_server/dashboard/assets/page-Bq0YxkLV.js +0 -1
  226. zenml/zen_server/dashboard/assets/page-Bs2F4eoD.js +0 -2
  227. zenml/zen_server/dashboard/assets/page-CHNxpz3n.js +0 -1
  228. zenml/zen_server/dashboard/assets/page-DgorQFqi.js +0 -1
  229. zenml/zen_server/dashboard/assets/page-K8ebxVIs.js +0 -1
  230. zenml/zen_server/dashboard/assets/page-TgCF0P_U.js +0 -1
  231. zenml/zen_server/dashboard/assets/page-ZnCEe-eK.js +0 -9
  232. zenml/zen_server/dashboard/assets/persist-D7HJNBWx.js +0 -1
  233. zenml/zen_server/dashboard/assets/plus-C8WOyCzt.js +0 -1
  234. zenml/zen_server/dashboard/assets/secrets-video-OBJ6irhH.svg +0 -21
  235. zenml/zen_server/dashboard/assets/stacks-video-7gfxpAq4.svg +0 -21
  236. /zenml/zen_server/{pipeline_deployment → template_execution}/__init__.py +0 -0
  237. /zenml/zen_server/{pipeline_deployment → template_execution}/workload_manager_interface.py +0 -0
  238. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/LICENSE +0 -0
  239. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/WHEEL +0 -0
  240. {zenml_nightly-0.62.0.dev20240729.dist-info → zenml_nightly-0.64.0.dev20240809.dist-info}/entry_points.txt +0 -0
zenml/cli/hub.py DELETED
@@ -1,1116 +0,0 @@
1
- # Copyright (c) ZenML GmbH 2023. All Rights Reserved.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at:
6
- #
7
- # https://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12
- # or implied. See the License for the specific language governing
13
- # permissions and limitations under the License.
14
- """CLI functionality to interact with the ZenML Hub."""
15
-
16
- import os
17
- import shutil
18
- import subprocess
19
- import sys
20
- import tempfile
21
- from importlib.util import find_spec
22
- from typing import Any, Dict, List, Optional, Tuple
23
-
24
- import click
25
-
26
- from zenml._hub.client import HubAPIError, HubClient
27
- from zenml._hub.constants import (
28
- ZENML_HUB_ADMIN_USERNAME,
29
- ZENML_HUB_INTERNAL_TAG_PREFIX,
30
- ZENML_HUB_VERIFIED_TAG,
31
- )
32
- from zenml._hub.utils import parse_plugin_name, plugin_display_name
33
- from zenml.analytics.enums import AnalyticsEvent
34
- from zenml.analytics.utils import track_handler
35
- from zenml.cli.cli import TagGroup, cli
36
- from zenml.cli.utils import declare, error, print_table, warning
37
- from zenml.enums import CliCategories
38
- from zenml.logger import get_logger
39
- from zenml.models import (
40
- HubPluginRequestModel,
41
- HubPluginResponseModel,
42
- PluginStatus,
43
- )
44
-
45
- logger = get_logger(__name__)
46
-
47
-
48
- @cli.group(cls=TagGroup, tag=CliCategories.HUB)
49
- def hub() -> None:
50
- """Interact with the ZenML Hub."""
51
-
52
-
53
- @hub.command("list")
54
- @click.option(
55
- "--all",
56
- "-a",
57
- is_flag=True,
58
- help="List all plugins, including those that are not available.",
59
- )
60
- @click.option(
61
- "--mine",
62
- "-m",
63
- is_flag=True,
64
- help="List only plugins that you own.",
65
- )
66
- @click.option(
67
- "--installed",
68
- "-i",
69
- is_flag=True,
70
- help="List only plugins that are installed.",
71
- )
72
- def list_plugins(
73
- all: bool = False, mine: bool = False, installed: bool = False
74
- ) -> None:
75
- """List all plugins available on the ZenML Hub.
76
-
77
- Args:
78
- all: Whether to list all plugins, including ones that are not available.
79
- mine: Whether to list only plugins that you own.
80
- installed: Whether to list only plugins that are installed.
81
- """
82
- client = HubClient()
83
- if mine and not client.auth_token:
84
- error(
85
- "You must be logged in to list your own plugins via --mine. "
86
- "Please run `zenml hub login` to login."
87
- )
88
- list_params: Dict[str, Any] = {"mine": mine}
89
- if not all:
90
- list_params["status"] = PluginStatus.AVAILABLE
91
- plugins = client.list_plugins(**list_params)
92
- if not plugins:
93
- declare("No plugins found.")
94
- if installed:
95
- plugins = [
96
- plugin
97
- for plugin in plugins
98
- if _is_plugin_installed(
99
- author=plugin.author, plugin_name=plugin.name
100
- )
101
- ]
102
- plugins_table = _format_plugins_table(plugins)
103
- print_table(plugins_table)
104
-
105
-
106
- def _format_plugins_table(
107
- plugins: List[HubPluginResponseModel],
108
- ) -> List[Dict[str, str]]:
109
- """Helper function to format a list of plugins into a table.
110
-
111
- Args:
112
- plugins: The list of plugins.
113
-
114
- Returns:
115
- The formatted table.
116
- """
117
- plugins_table = []
118
- for plugin in sorted(plugins, key=_sort_plugin_key_fn):
119
- if _is_plugin_installed(author=plugin.author, plugin_name=plugin.name):
120
- installed_icon = ":white_check_mark:"
121
- else:
122
- installed_icon = ":x:"
123
- if plugin.status == PluginStatus.AVAILABLE:
124
- status_icon = ":white_check_mark:"
125
- elif plugin.status == PluginStatus.PENDING:
126
- status_icon = ":hourglass:"
127
- else:
128
- status_icon = ":x:"
129
-
130
- display_name = plugin_display_name(
131
- name=plugin.name,
132
- version=plugin.version,
133
- author=plugin.author,
134
- )
135
- display_data: Dict[str, str] = {
136
- "PLUGIN": display_name,
137
- "STATUS": status_icon,
138
- "INSTALLED": installed_icon,
139
- "MODULE": _get_plugin_module(
140
- author=plugin.author, plugin_name=plugin.name
141
- ),
142
- "PACKAGE NAME": plugin.package_name or "",
143
- "REPOSITORY URL": plugin.repository_url,
144
- }
145
- plugins_table.append(display_data)
146
- return plugins_table
147
-
148
-
149
- def _sort_plugin_key_fn(
150
- plugin: HubPluginResponseModel,
151
- ) -> Tuple[str, str, str]:
152
- """Helper function to sort plugins by name, version and author.
153
-
154
- Args:
155
- plugin: The plugin to sort.
156
-
157
- Returns:
158
- A tuple of the plugin's author, name and version.
159
- """
160
- if plugin.author == ZENML_HUB_ADMIN_USERNAME:
161
- return "0", plugin.name, plugin.version # Sort admin plugins first
162
- return plugin.author, plugin.name, plugin.version
163
-
164
-
165
- @hub.command("install")
166
- @click.argument("plugin_name", type=str, required=True)
167
- @click.option(
168
- "--upgrade",
169
- "-u",
170
- is_flag=True,
171
- help="Upgrade the plugin if it is already installed.",
172
- )
173
- @click.option(
174
- "--no-deps",
175
- "--no-dependencies",
176
- is_flag=True,
177
- help="Do not install dependencies of the plugin.",
178
- )
179
- @click.option(
180
- "--yes",
181
- "-y",
182
- is_flag=True,
183
- help="Do not ask for confirmation before installing.",
184
- )
185
- def install_plugin(
186
- plugin_name: str,
187
- upgrade: bool = False,
188
- no_deps: bool = False,
189
- yes: bool = False,
190
- ) -> None:
191
- """Install a plugin from the ZenML Hub.
192
-
193
- Args:
194
- plugin_name: Name of the plugin.
195
- upgrade: Whether to upgrade the plugin if it is already installed.
196
- no_deps: If set, dependencies of the plugin will not be installed.
197
- yes: If set, no confirmation will be asked for before installing.
198
- """
199
- with track_handler(
200
- event=AnalyticsEvent.ZENML_HUB_PLUGIN_INSTALL,
201
- ) as analytics_handler:
202
- client = HubClient()
203
- analytics_handler.metadata["hub_url"] = client.url
204
- author, plugin_name, plugin_version = parse_plugin_name(plugin_name)
205
- analytics_handler.metadata["plugin_name"] = plugin_name
206
- analytics_handler.metadata["plugin_author"] = author
207
- display_name = plugin_display_name(plugin_name, plugin_version, author)
208
-
209
- # Get plugin from hub
210
- plugin = client.get_plugin(
211
- name=plugin_name,
212
- version=plugin_version,
213
- author=author,
214
- )
215
- if not plugin:
216
- error(f"Could not find plugin '{display_name}' on the hub.")
217
- analytics_handler.metadata["plugin_version"] = plugin.version
218
-
219
- # Check if plugin can be installed
220
- index_url = plugin.index_url
221
- package_name = plugin.package_name
222
- if not index_url or not package_name:
223
- error(
224
- f"Plugin '{display_name}' is not available for installation."
225
- )
226
-
227
- # Check if plugin is already installed
228
- if (
229
- _is_plugin_installed(author=plugin.author, plugin_name=plugin.name)
230
- and not upgrade
231
- ):
232
- declare(f"Plugin '{plugin_name}' is already installed.")
233
- return
234
-
235
- # Show a warning if the plugin is not official or verified
236
- _is_zenml_plugin = plugin.author == ZENML_HUB_ADMIN_USERNAME
237
- _is_verified = plugin.tags and ZENML_HUB_VERIFIED_TAG in plugin.tags
238
- if not _is_zenml_plugin and not _is_verified:
239
- warning(
240
- f"Plugin '{display_name}' was not verified by ZenML and may "
241
- "contain arbitrary code. Please check the source code before "
242
- "installing to make sure it does what you expect."
243
- )
244
-
245
- # Install plugin requirements
246
- if plugin.requirements:
247
- requirements_str = " ".join(f"'{r}'" for r in plugin.requirements)
248
- install_requirements = False
249
- if not no_deps and not yes:
250
- install_requirements = click.confirm(
251
- f"Plugin '{display_name}' requires the following "
252
- f"packages to be installed: {requirements_str}. "
253
- f"Do you want to install them now?"
254
- )
255
- if yes or install_requirements:
256
- declare(
257
- f"Installing requirements for plugin '{display_name}': "
258
- f"{requirements_str}..."
259
- )
260
- requirements_install_call = [
261
- sys.executable,
262
- "-m",
263
- "pip",
264
- "install",
265
- *list(plugin.requirements),
266
- "--upgrade",
267
- ]
268
- subprocess.check_call(requirements_install_call)
269
- declare(
270
- f"Successfully installed requirements for plugin "
271
- f"'{display_name}'."
272
- )
273
- else:
274
- warning(
275
- f"Requirements for plugin '{display_name}' were not "
276
- "installed. This might lead to errors in the future if the "
277
- "requirements are not installed manually."
278
- )
279
-
280
- # pip install the wheel
281
- declare(
282
- f"Installing plugin '{display_name}' from "
283
- f"{index_url}{package_name}..."
284
- )
285
- plugin_install_call = [
286
- sys.executable,
287
- "-m",
288
- "pip",
289
- "install",
290
- "--index-url",
291
- index_url,
292
- package_name,
293
- "--no-deps", # we already installed the requirements above
294
- "--upgrade", # we already checked if the plugin is installed above
295
- ]
296
- subprocess.check_call(plugin_install_call)
297
- declare(f"Successfully installed plugin '{display_name}'.")
298
-
299
-
300
- @hub.command("uninstall")
301
- @click.argument("plugin_name", type=str, required=True)
302
- def uninstall_plugin(plugin_name: str) -> None:
303
- """Uninstall a ZenML Hub plugin.
304
-
305
- Args:
306
- plugin_name: Name of the plugin.
307
- """
308
- with track_handler(
309
- event=AnalyticsEvent.ZENML_HUB_PLUGIN_UNINSTALL,
310
- ) as analytics_handler:
311
- client = HubClient()
312
- analytics_handler.metadata["hub_url"] = client.url
313
- author, plugin_name, plugin_version = parse_plugin_name(plugin_name)
314
- analytics_handler.metadata["plugin_name"] = plugin_name
315
- analytics_handler.metadata["plugin_author"] = author
316
- display_name = plugin_display_name(plugin_name, plugin_version, author)
317
-
318
- # Get plugin from hub
319
- plugin = client.get_plugin(
320
- name=plugin_name,
321
- version=plugin_version,
322
- author=author,
323
- )
324
- if not plugin:
325
- error(f"Could not find plugin '{display_name}' on the hub.")
326
- analytics_handler.metadata["plugin_version"] = plugin.version
327
-
328
- # Check if plugin can be uninstalled
329
- package_name = plugin.package_name
330
- if not package_name:
331
- error(
332
- f"Plugin '{display_name}' is not available for uninstallation."
333
- )
334
-
335
- # pip uninstall the wheel
336
- declare(f"Uninstalling plugin '{display_name}'...")
337
- subprocess.check_call(
338
- [sys.executable, "-m", "pip", "uninstall", package_name, "-y"]
339
- )
340
- declare(f"Successfully uninstalled plugin '{display_name}'.")
341
-
342
-
343
- @hub.command("clone")
344
- @click.argument("plugin_name", type=str, required=True)
345
- @click.option(
346
- "--output_dir",
347
- "-o",
348
- type=str,
349
- help="Output directory to clone the plugin to.",
350
- )
351
- def clone_plugin(
352
- plugin_name: str,
353
- output_dir: Optional[str] = None,
354
- ) -> None:
355
- """Clone the source code of a ZenML Hub plugin.
356
-
357
- Args:
358
- plugin_name: Name of the plugin.
359
- output_dir: Output directory to clone the plugin to. If not specified,
360
- the plugin will be cloned to a directory with the same name as the
361
- plugin in the current working directory.
362
- """
363
- from zenml.utils.git_utils import clone_git_repository
364
-
365
- with track_handler(
366
- event=AnalyticsEvent.ZENML_HUB_PLUGIN_CLONE,
367
- ) as analytics_handler:
368
- client = HubClient()
369
- analytics_handler.metadata["hub_url"] = client.url
370
- author, plugin_name, plugin_version = parse_plugin_name(plugin_name)
371
- analytics_handler.metadata["plugin_name"] = plugin_name
372
- analytics_handler.metadata["plugin_author"] = author
373
- display_name = plugin_display_name(plugin_name, plugin_version, author)
374
-
375
- # Get plugin from hub
376
- plugin = client.get_plugin(
377
- name=plugin_name,
378
- version=plugin_version,
379
- author=author,
380
- )
381
- if not plugin:
382
- error(f"Could not find plugin '{display_name}' on the hub.")
383
- analytics_handler.metadata["plugin_version"] = plugin.version
384
-
385
- # Clone the source repo to a temp dir, then move the plugin subdir out
386
- repo_url = plugin.repository_url
387
- subdir = plugin.repository_subdirectory
388
- commit = plugin.repository_commit
389
- if output_dir is None:
390
- output_dir = os.path.join(os.getcwd(), plugin_name)
391
- declare(f"Cloning plugin '{display_name}' to {output_dir}...")
392
- with tempfile.TemporaryDirectory() as tmp_dir:
393
- try:
394
- clone_git_repository(
395
- url=repo_url, to_path=tmp_dir, commit=commit
396
- )
397
- except RuntimeError:
398
- error(
399
- f"Could not find commit '{commit}' in repository "
400
- f"'{repo_url}' of plugin '{display_name}'. This might "
401
- "happen if the owner of the plugin has force-pushed to the "
402
- "plugin repository or taken it down. Please report this "
403
- "plugin version in the ZenML Hub or via Slack."
404
- )
405
- plugin_dir = os.path.join(tmp_dir, subdir or "")
406
- shutil.move(plugin_dir, output_dir)
407
- declare(f"Successfully Cloned plugin '{display_name}'.")
408
-
409
-
410
- @hub.command("login")
411
- @click.option(
412
- "--github",
413
- "-g",
414
- is_flag=True,
415
- help="Login via GitHub.",
416
- )
417
- @click.option(
418
- "--email",
419
- "-e",
420
- type=str,
421
- help="Login via ZenML Hub account using this email address.",
422
- )
423
- @click.option(
424
- "--password",
425
- "-p",
426
- type=str,
427
- help="Password of the ZenML Hub account.",
428
- )
429
- def login(
430
- github: bool = False,
431
- email: Optional[str] = None,
432
- password: Optional[str] = None,
433
- ) -> None:
434
- """Login to the ZenML Hub.
435
-
436
- Args:
437
- github: Login via GitHub.
438
- email: Login via ZenML Hub account using this email address.
439
- password: Password of the ZenML Hub account. Only used if `email` is
440
- specified.
441
- """
442
- if github:
443
- _login_via_github()
444
- elif email:
445
- if not password:
446
- password = click.prompt("Password", type=str, hide_input=True)
447
- _login_via_zenml_hub(email, password)
448
- else:
449
- declare(
450
- "You can either login via your ZenML Hub account or via GitHub."
451
- )
452
- confirmation = click.confirm("Login via ZenML Hub account?")
453
- if confirmation:
454
- _login_via_zenml_hub()
455
- else:
456
- _login_via_github()
457
-
458
-
459
- def _login_via_zenml_hub(
460
- email: Optional[str] = None, password: Optional[str] = None
461
- ) -> None:
462
- """Login via ZenML Hub email and password.
463
-
464
- Args:
465
- email: Login via ZenML Hub account using this email address.
466
- password: Password of the ZenML Hub account. Only used if `email` is
467
- specified.
468
- """
469
- client = HubClient()
470
- if not email or not password:
471
- declare("Please enter your ZenML Hub credentials.")
472
- while not email:
473
- email = click.prompt("Email", type=str)
474
- while not password:
475
- password = click.prompt("Password", type=str, hide_input=True)
476
- try:
477
- client.login(email, password)
478
- me = client.get_me()
479
- if me:
480
- declare(f"Successfully logged in as: {me.username} ({me.email})!")
481
- return
482
- error("Could not retrieve user information from the ZenML Hub.")
483
- except HubAPIError as e:
484
- error(f"Could not login to the ZenML Hub: {e}")
485
-
486
-
487
- def _login_via_github() -> None:
488
- """Login via GitHub."""
489
- client = HubClient()
490
- try:
491
- login_url = client.get_github_login_url()
492
- except HubAPIError as e:
493
- error(f"Could not retrieve GitHub login URL: {e}")
494
- declare(f"Please open the following URL in your browser: {login_url}")
495
- auth_token = click.prompt("Please enter your auth token", type=str)
496
- client.set_auth_token(auth_token)
497
- declare("Successfully logged in to the ZenML Hub.")
498
-
499
-
500
- @hub.command("logout")
501
- def logout() -> None:
502
- """Logout from the ZenML Hub."""
503
- client = HubClient()
504
- client.set_auth_token(None)
505
- declare("Successfully logged out from the ZenML Hub.")
506
-
507
-
508
- @hub.command("submit")
509
- @click.option(
510
- "--plugin_name",
511
- "-n",
512
- type=str,
513
- help=(
514
- "Name of the plugin to submit. If not provided, the name will be asked "
515
- "for interactively."
516
- ),
517
- )
518
- @click.option(
519
- "--version",
520
- "-v",
521
- type=str,
522
- help=(
523
- "Version of the plugin to submit. Can only be set if the plugin "
524
- "already exists. If not provided, the version will be auto-incremented."
525
- ),
526
- )
527
- @click.option(
528
- "--release_notes",
529
- "-r",
530
- type=str,
531
- help="Release notes for the plugin version.",
532
- )
533
- @click.option(
534
- "--description",
535
- "-d",
536
- type=str,
537
- help="Description of the plugin.",
538
- )
539
- @click.option(
540
- "--repository_url",
541
- "-u",
542
- type=str,
543
- help="URL to the public Git repository containing the plugin source code.",
544
- )
545
- @click.option(
546
- "--repository_subdir",
547
- "-s",
548
- type=str,
549
- help="Subdirectory of the repository containing the plugin source code.",
550
- )
551
- @click.option(
552
- "--repository_branch",
553
- "-b",
554
- type=str,
555
- help="Branch to checkout from the repository.",
556
- )
557
- @click.option(
558
- "--repository_commit",
559
- "-c",
560
- type=str,
561
- help="Commit to checkout from the repository. Overrides --branch.",
562
- )
563
- @click.option(
564
- "tags",
565
- "--tag",
566
- "-t",
567
- type=str,
568
- multiple=True,
569
- )
570
- @click.option(
571
- "--interactive",
572
- "-i",
573
- is_flag=True,
574
- help="Run the command in interactive mode.",
575
- )
576
- def submit_plugin(
577
- plugin_name: Optional[str] = None,
578
- version: Optional[str] = None,
579
- release_notes: Optional[str] = None,
580
- description: Optional[str] = None,
581
- repository_url: Optional[str] = None,
582
- repository_subdir: Optional[str] = None,
583
- repository_branch: Optional[str] = None,
584
- repository_commit: Optional[str] = None,
585
- tags: Optional[List[str]] = None,
586
- interactive: bool = False,
587
- ) -> None:
588
- """Submit a plugin to the ZenML Hub.
589
-
590
- Args:
591
- plugin_name: Name of the plugin to submit. Needs to be set unless
592
- interactive mode is enabled.
593
- version: Version of the plugin to submit. Can only be set if the plugin
594
- already exists. If not provided, the version will be
595
- auto-incremented.
596
- release_notes: Release notes for the plugin version.
597
- description: Description of the plugin.
598
- repository_url: URL to the public Git repository containing the plugin
599
- source code. Needs to be set unless interactive mode is enabled.
600
- repository_subdir: Subdirectory of the repository containing the plugin
601
- source code.
602
- repository_branch: Branch to checkout from the repository.
603
- repository_commit: Commit to checkout from the repository. Overrides
604
- `repository_branch`.
605
- tags: Tags to add to the plugin.
606
- interactive: Whether to run the command in interactive mode, asking for
607
- missing or invalid parameters.
608
- """
609
- with track_handler(
610
- event=AnalyticsEvent.ZENML_HUB_PLUGIN_SUBMIT,
611
- ) as analytics_handler:
612
- client = HubClient()
613
- analytics_handler.metadata["hub_url"] = client.url
614
-
615
- # Validate that the user is logged in
616
- if not client.auth_token:
617
- error(
618
- "You must be logged in to contribute a plugin to the Hub. "
619
- "Please run `zenml hub login` to login."
620
- )
621
- me = client.get_me()
622
- if not me:
623
- error("Could not retrieve user information from the ZenML Hub.")
624
- if not me.username:
625
- error(
626
- "Your ZenML Hub account does not have a username yet. Please "
627
- "set a username in your account settings and try again."
628
- )
629
- username = me.username
630
-
631
- # Validate the plugin name and check if it exists
632
- plugin_name, plugin_exists = _validate_plugin_name(
633
- client=client,
634
- plugin_name=plugin_name,
635
- username=username,
636
- interactive=interactive,
637
- )
638
-
639
- # If the plugin exists, ask for version and release notes in
640
- # interactive mode.
641
- if plugin_exists and interactive:
642
- if not version:
643
- declare(
644
- "You are about to create a new version of plugin "
645
- f"'{plugin_name}'. By default, this will increment the "
646
- "minor version of the plugin. If you want to specify a "
647
- "different version, you can do so below. In that case, "
648
- "make sure that the version is of shape '<int>.<int>' and "
649
- "is higher than the current latest version of the plugin."
650
- )
651
- version = click.prompt("(Optional) plugin version", default="")
652
- if not release_notes:
653
- declare(
654
- f"You are about to create a new version of plugin "
655
- f"'{plugin_name}'. You can optionally provide release "
656
- "notes for this version below."
657
- )
658
- release_notes = click.prompt(
659
- "(Optional) release notes", default=""
660
- )
661
-
662
- # Clone the repo and validate the commit / branch / subdir / structure
663
- (
664
- repository_url,
665
- repository_commit,
666
- repository_branch,
667
- repository_subdir,
668
- ) = _validate_repository(
669
- url=repository_url,
670
- commit=repository_commit,
671
- branch=repository_branch,
672
- subdir=repository_subdir,
673
- interactive=interactive,
674
- )
675
-
676
- # In interactive mode, ask for a description if none is provided
677
- if interactive and not description:
678
- declare(
679
- "You can optionally provide a description for your plugin "
680
- "below. If not set, the first line of your README.md will "
681
- "be used."
682
- )
683
- description = click.prompt(
684
- "(Optional) plugin description", default=""
685
- )
686
-
687
- # Validate the tags
688
- if tags:
689
- tags = _validate_tags(tags=tags, interactive=interactive)
690
- else:
691
- tags = []
692
-
693
- # Make a create request to the hub
694
- plugin_request = HubPluginRequestModel(
695
- name=plugin_name,
696
- description=description,
697
- version=version,
698
- release_notes=release_notes,
699
- repository_url=repository_url,
700
- repository_subdirectory=repository_subdir,
701
- repository_branch=repository_branch,
702
- repository_commit=repository_commit,
703
- tags=tags,
704
- )
705
- plugin_response = client.create_plugin(
706
- plugin_request=plugin_request,
707
- )
708
-
709
- # Stream the build logs
710
- plugin_name = plugin_response.name
711
- plugin_version = plugin_response.version
712
- declare(
713
- "Thanks for submitting your plugin to the ZenML Hub. The plugin is "
714
- "now being built into an installable package. This may take a few "
715
- "minutes. To view the build logs, run "
716
- f"`zenml hub logs {username}/{plugin_name}:{plugin_version}`."
717
- )
718
-
719
-
720
- def _validate_plugin_name(
721
- client: HubClient,
722
- plugin_name: Optional[str],
723
- username: str,
724
- interactive: bool,
725
- ) -> Tuple[str, bool]:
726
- """Validate that the plugin name is provided and available.
727
-
728
- Args:
729
- client: The Hub client used to check if the plugin name is available.
730
- plugin_name: The plugin name to validate.
731
- username: The username of the current user.
732
- interactive: Whether to run in interactive mode.
733
-
734
- Returns:
735
- The validated plugin name, and whether the plugin already exists.
736
- """
737
- # Make sure the plugin name is provided.
738
- while not plugin_name:
739
- if not interactive:
740
- error("Plugin name not provided.")
741
- declare("Please enter a name for the plugin.")
742
- plugin_name = click.prompt("Plugin name")
743
-
744
- existing_plugin = client.get_plugin(name=plugin_name, author=username)
745
- return plugin_name, bool(existing_plugin)
746
-
747
-
748
- def _validate_repository(
749
- url: Optional[str],
750
- commit: Optional[str],
751
- branch: Optional[str],
752
- subdir: Optional[str],
753
- interactive: bool,
754
- ) -> Tuple[str, Optional[str], Optional[str], Optional[str]]:
755
- """Validate the provided repository arguments.
756
-
757
- Args:
758
- url: The URL to the repository to clone.
759
- commit: The commit to checkout.
760
- branch: The branch to checkout. Will be ignored if commit is provided.
761
- subdir: The subdirectory in which the plugin is located.
762
- interactive: Whether to run in interactive mode.
763
-
764
- Returns:
765
- The validated URL, commit, branch, and subdirectory.
766
- """
767
- from zenml.utils.git_utils import clone_git_repository
768
-
769
- while True:
770
- # Make sure the repository URL is provided.
771
- if not url:
772
- if not interactive:
773
- error("Repository URL not provided.")
774
- declare(
775
- "Please enter the URL to the public Git repository containing "
776
- "the plugin source code."
777
- )
778
- url = click.prompt("Repository URL")
779
- assert url is not None
780
-
781
- # In interactive mode, ask for the branch and commit if not provided
782
- if interactive and not branch and not commit:
783
- confirmation = click.confirm(
784
- "Do you want to use the latest commit from the 'main' branch "
785
- "of the repository?"
786
- )
787
- if not confirmation:
788
- confirmation = click.confirm(
789
- "You can either use a specific commit or the latest commit "
790
- "from one of your branches. Do you want to use a specific "
791
- "commit?"
792
- )
793
- if confirmation:
794
- declare("Please enter the SHA of the commit.")
795
- commit = click.prompt("Repository commit")
796
- branch = None
797
- else:
798
- declare("Please enter the name of a branch.")
799
- branch = click.prompt("Repository branch")
800
- commit = None
801
-
802
- try:
803
- # Check if the URL/branch/commit are valid.
804
- with tempfile.TemporaryDirectory() as tmp_dir:
805
- clone_git_repository(
806
- url=url,
807
- commit=commit,
808
- branch=branch,
809
- to_path=tmp_dir,
810
- )
811
-
812
- # Check if the subdir exists and has the correct structure.
813
- subdir = _validate_repository_subdir(
814
- repository_subdir=subdir,
815
- repo_path=tmp_dir,
816
- interactive=interactive,
817
- )
818
-
819
- return url, commit, branch, subdir
820
-
821
- except RuntimeError:
822
- repo_display_name = f"'{url}'"
823
- suggestion = "Please enter a valid repository URL"
824
- if commit:
825
- repo_display_name += f" (commit '{commit}')"
826
- suggestion += " and make sure the commit exists."
827
- elif branch:
828
- repo_display_name += f" (branch '{branch}')"
829
- suggestion += " and make sure the branch exists."
830
- else:
831
- suggestion += " and make sure the 'main' branch exists."
832
- msg = f"Could not clone repository from URL {repo_display_name}. "
833
- if not interactive:
834
- error(msg + suggestion)
835
- declare(msg + suggestion)
836
- url, branch, commit = None, None, None
837
-
838
-
839
- def _validate_repository_subdir(
840
- repository_subdir: Optional[str], repo_path: str, interactive: bool
841
- ) -> Optional[str]:
842
- """Validate the provided repository subdirectory.
843
-
844
- Args:
845
- repository_subdir: The subdirectory to validate.
846
- repo_path: The path to the repository to validate the subdirectory in.
847
- interactive: Whether to run in interactive mode.
848
-
849
- Returns:
850
- The validated subdirectory.
851
- """
852
- while True:
853
- # In interactive mode, ask for the subdirectory if not provided
854
- if interactive and not repository_subdir:
855
- confirmation = click.confirm(
856
- "Is the plugin source code in the root of the repository?"
857
- )
858
- if not confirmation:
859
- declare(
860
- "Please enter the subdirectory of the repository "
861
- "containing the plugin source code."
862
- )
863
- repository_subdir = click.prompt("Repository subdirectory")
864
-
865
- # If a subdir was provided, make sure it exists
866
- if repository_subdir:
867
- subdir_path = os.path.join(repo_path, repository_subdir)
868
- if not os.path.exists(subdir_path):
869
- if not interactive:
870
- error("Repository subdirectory does not exist.")
871
- declare(
872
- f"Subdirectory '{repository_subdir}' does not exist in the "
873
- f"repository."
874
- )
875
- declare("Please enter a valid subdirectory.")
876
- repository_subdir = click.prompt(
877
- "Repository subdirectory", default=""
878
- )
879
- continue
880
-
881
- # Check if the plugin structure is valid.
882
- if repository_subdir:
883
- plugin_path = os.path.join(repo_path, repository_subdir)
884
- else:
885
- plugin_path = repo_path
886
- try:
887
- _validate_repository_structure(plugin_path)
888
- return repository_subdir
889
- except ValueError as e:
890
- msg = (
891
- f"Plugin code structure at {repository_subdir} is invalid: "
892
- f"{str(e)}"
893
- )
894
- if not interactive:
895
- error(str(e))
896
- declare(msg)
897
- declare("Please enter a valid subdirectory.")
898
- repository_subdir = click.prompt("Repository subdirectory")
899
-
900
-
901
- def _validate_repository_structure(plugin_root: str) -> None:
902
- """Validate the repository structure of a submitted ZenML Hub plugin.
903
-
904
- We expect the following structure:
905
- - src/__init__.py
906
- - README.md
907
- - requirements.txt
908
- - (Optional) logo.png
909
-
910
- Args:
911
- plugin_root: Root directory of the plugin.
912
-
913
- Raises:
914
- ValueError: If the repo does not have the correct structure.
915
- """
916
- # src/__init__.py exists.
917
- src_path = os.path.join(plugin_root, "src")
918
- if not os.path.exists(src_path):
919
- raise ValueError("src/ not found")
920
- init_path = os.path.join(src_path, "__init__.py")
921
- if not os.path.exists(init_path):
922
- raise ValueError("src/__init__.py not found")
923
-
924
- # README.md exists.
925
- readme_path = os.path.join(plugin_root, "README.md")
926
- if not os.path.exists(readme_path):
927
- raise ValueError("README.md not found")
928
-
929
- # requirements.txt exists.
930
- requirements_path = os.path.join(plugin_root, "requirements.txt")
931
- if not os.path.exists(requirements_path):
932
- raise ValueError("requirements.txt not found")
933
-
934
-
935
- def _validate_tags(tags: List[str], interactive: bool) -> List[str]:
936
- """Validate the provided tags.
937
-
938
- Args:
939
- tags: The tags to validate.
940
- interactive: Whether to run in interactive mode.
941
-
942
- Returns:
943
- The validated tags.
944
- """
945
- if not tags:
946
- if not interactive:
947
- return []
948
-
949
- # In interactive mode, ask for tags if none were provided.
950
- return _ask_for_tags()
951
-
952
- # If tags were provided, print a warning if any of them is invalid.
953
- for tag in tags:
954
- if tag.startswith(ZENML_HUB_INTERNAL_TAG_PREFIX):
955
- warning(
956
- f"Tag '{tag}' will be ignored because it starts with "
957
- f"disallowed prefix '{ZENML_HUB_INTERNAL_TAG_PREFIX}'."
958
- )
959
- return tags
960
-
961
-
962
- def _ask_for_tags() -> List[str]:
963
- """Repeatedly ask the user for tags to assign to the plugin.
964
-
965
- Returns:
966
- A list of tags.
967
- """
968
- tags: List[str] = []
969
- while True:
970
- tag = click.prompt(
971
- "(Optional) enter tags you want to assign to the plugin.",
972
- default="",
973
- )
974
- if not tag:
975
- return tags
976
- if tag.startswith(ZENML_HUB_INTERNAL_TAG_PREFIX):
977
- warning(
978
- "User-defined tags may not start with "
979
- f"'{ZENML_HUB_INTERNAL_TAG_PREFIX}'."
980
- )
981
- else:
982
- tags.append(tag)
983
-
984
-
985
- @hub.command("submit-batch")
986
- @click.argument(
987
- "config", type=click.Path(exists=True, dir_okay=False), required=True
988
- )
989
- def batch_submit(config: str) -> None:
990
- """Batch submit plugins to the ZenML Hub.
991
-
992
- WARNING: This command is intended for advanced users only. It does not
993
- perform any client-side validation which might lead to server-side HTTP
994
- errors that are hard to debug if the config file you specify is invalid or
995
- contains invalid plugin definitions. When in doubt, use the
996
- `zenml hub submit` command instead.
997
-
998
- Args:
999
- config: Path to the config file. The config file is expected to be a
1000
- list of plugin definitions. Each plugin definition must be a dict
1001
- with keys and values matching the fields of `HubPluginRequestModel`:
1002
- ```yaml
1003
- - name: str
1004
- version: str
1005
- release_notes: str
1006
- description: str
1007
- repository_url: str
1008
- repository_subdirectory: str
1009
- repository_branch: str
1010
- repository_commit: str
1011
- logo_url: str
1012
- tags:
1013
- - str
1014
- - ...
1015
- - ...
1016
- ```
1017
- """
1018
- from pydantic import ValidationError
1019
-
1020
- from zenml.utils.yaml_utils import read_yaml
1021
-
1022
- client = HubClient()
1023
- config = read_yaml(config)
1024
- if not isinstance(config, list):
1025
- error("Config file must be a list of plugin definitions.")
1026
- declare(f"Submitting {len(config)} plugins to the hub...")
1027
- for plugin_dict in config:
1028
- try:
1029
- assert isinstance(plugin_dict, dict)
1030
- plugin_request = HubPluginRequestModel(**plugin_dict)
1031
- plugin = client.create_plugin(plugin_request=plugin_request)
1032
- except (AssertionError, ValidationError, HubAPIError) as e:
1033
- warning(f"Could not submit plugin: {str(e)}")
1034
- continue
1035
- display_name = plugin_display_name(
1036
- name=plugin.name,
1037
- version=plugin.version,
1038
- author=plugin.author,
1039
- )
1040
- declare(f"Submitted plugin '{display_name}' to the hub.")
1041
-
1042
-
1043
- @hub.command("logs")
1044
- @click.argument("plugin_name", type=str, required=True)
1045
- def get_logs(plugin_name: str) -> None:
1046
- """Get the build logs of a ZenML Hub plugin.
1047
-
1048
- Args:
1049
- plugin_name: Name of the plugin.
1050
- """
1051
- client = HubClient()
1052
- author, plugin_name, plugin_version = parse_plugin_name(plugin_name)
1053
- display_name = plugin_display_name(plugin_name, plugin_version, author)
1054
-
1055
- # Get the plugin from the hub
1056
- plugin = client.get_plugin(
1057
- name=plugin_name,
1058
- version=plugin_version,
1059
- author=author,
1060
- )
1061
- if not plugin:
1062
- error(f"Could not find plugin '{display_name}' on the hub.")
1063
-
1064
- if plugin.status == PluginStatus.PENDING:
1065
- error(
1066
- f"Plugin '{display_name}' is still being built. Please try "
1067
- "again later."
1068
- )
1069
-
1070
- if not plugin.build_logs:
1071
- declare(
1072
- f"Plugin '{display_name}' finished building, but no logs "
1073
- "were found."
1074
- )
1075
- return
1076
-
1077
- for line in plugin.build_logs.splitlines():
1078
- declare(line)
1079
-
1080
-
1081
- # GENERAL HELPER FUNCTIONS
1082
-
1083
-
1084
- def _is_plugin_installed(author: str, plugin_name: str) -> bool:
1085
- """Helper function to check if a plugin is installed.
1086
-
1087
- Args:
1088
- author: The author of the plugin.
1089
- plugin_name: The name of the plugin.
1090
-
1091
- Returns:
1092
- Whether the plugin is installed.
1093
- """
1094
- module_name = _get_plugin_module(author=author, plugin_name=plugin_name)
1095
- try:
1096
- spec = find_spec(module_name)
1097
- return spec is not None
1098
- except ModuleNotFoundError:
1099
- return False
1100
-
1101
-
1102
- def _get_plugin_module(author: str, plugin_name: str) -> str:
1103
- """Helper function to get the module name of a plugin.
1104
-
1105
- Args:
1106
- author: The author of the plugin.
1107
- plugin_name: The name of the plugin.
1108
-
1109
- Returns:
1110
- The module name of the plugin.
1111
- """
1112
- module_name = "zenml.hub"
1113
- if author != ZENML_HUB_ADMIN_USERNAME:
1114
- module_name += f".{author}"
1115
- module_name += f".{plugin_name}"
1116
- return module_name