zenml-nightly 0.60.0.dev20240627__py3-none-any.whl → 0.61.0.dev20240711__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. README.md +30 -9
  2. RELEASE_NOTES.md +34 -0
  3. zenml/VERSION +1 -1
  4. zenml/analytics/enums.py +3 -0
  5. zenml/cli/__init__.py +28 -0
  6. zenml/cli/artifact.py +1 -2
  7. zenml/cli/integration.py +9 -8
  8. zenml/cli/server.py +6 -0
  9. zenml/cli/stack.py +797 -39
  10. zenml/cli/stack_components.py +7 -0
  11. zenml/cli/text_utils.py +35 -1
  12. zenml/cli/utils.py +127 -10
  13. zenml/client.py +23 -14
  14. zenml/config/docker_settings.py +8 -5
  15. zenml/constants.py +5 -1
  16. zenml/container_registries/base_container_registry.py +1 -0
  17. zenml/enums.py +6 -0
  18. zenml/event_hub/event_hub.py +5 -8
  19. zenml/integrations/aws/__init__.py +1 -0
  20. zenml/integrations/azure/__init__.py +1 -0
  21. zenml/integrations/deepchecks/__init__.py +1 -0
  22. zenml/integrations/discord/__init__.py +1 -0
  23. zenml/integrations/evidently/__init__.py +1 -0
  24. zenml/integrations/facets/__init__.py +1 -0
  25. zenml/integrations/feast/__init__.py +1 -0
  26. zenml/integrations/gcp/__init__.py +3 -1
  27. zenml/integrations/gcp/service_connectors/gcp_service_connector.py +203 -44
  28. zenml/integrations/huggingface/__init__.py +1 -0
  29. zenml/integrations/integration.py +24 -0
  30. zenml/integrations/kubeflow/__init__.py +3 -0
  31. zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +1 -1
  32. zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +0 -1
  33. zenml/integrations/kubernetes/__init__.py +3 -1
  34. zenml/integrations/kubernetes/orchestrators/kube_utils.py +4 -1
  35. zenml/integrations/label_studio/annotators/label_studio_annotator.py +1 -0
  36. zenml/integrations/langchain/__init__.py +1 -0
  37. zenml/integrations/mlflow/__init__.py +3 -1
  38. zenml/integrations/neural_prophet/__init__.py +1 -0
  39. zenml/integrations/polars/__init__.py +1 -0
  40. zenml/integrations/prodigy/__init__.py +1 -0
  41. zenml/integrations/pycaret/__init__.py +6 -0
  42. zenml/integrations/registry.py +37 -0
  43. zenml/integrations/s3/artifact_stores/s3_artifact_store.py +17 -6
  44. zenml/integrations/seldon/__init__.py +1 -0
  45. zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +1 -0
  46. zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +2 -2
  47. zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +1 -1
  48. zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
  49. zenml/integrations/skypilot_aws/__init__.py +2 -1
  50. zenml/integrations/skypilot_azure/__init__.py +1 -1
  51. zenml/integrations/skypilot_gcp/__init__.py +1 -1
  52. zenml/integrations/skypilot_lambda/__init__.py +1 -1
  53. zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +1 -1
  54. zenml/integrations/slack/__init__.py +1 -0
  55. zenml/integrations/tekton/__init__.py +1 -0
  56. zenml/integrations/tensorboard/__init__.py +0 -1
  57. zenml/integrations/tensorflow/__init__.py +18 -6
  58. zenml/integrations/wandb/__init__.py +1 -0
  59. zenml/models/__init__.py +9 -0
  60. zenml/models/v2/core/component.py +18 -0
  61. zenml/models/v2/core/model.py +1 -2
  62. zenml/models/v2/core/service_connector.py +17 -0
  63. zenml/models/v2/core/stack.py +31 -0
  64. zenml/models/v2/misc/full_stack.py +97 -0
  65. zenml/models/v2/misc/stack_deployment.py +66 -0
  66. zenml/new/pipelines/pipeline.py +1 -1
  67. zenml/orchestrators/input_utils.py +3 -6
  68. zenml/stack/stack.py +3 -6
  69. zenml/stack_deployments/__init__.py +14 -0
  70. zenml/stack_deployments/aws_stack_deployment.py +289 -0
  71. zenml/stack_deployments/stack_deployment.py +130 -0
  72. zenml/stack_deployments/utils.py +40 -0
  73. zenml/utils/function_utils.py +1 -1
  74. zenml/utils/pagination_utils.py +7 -5
  75. zenml/utils/pipeline_docker_image_builder.py +97 -68
  76. zenml/utils/pydantic_utils.py +6 -5
  77. zenml/zen_server/cloud_utils.py +18 -3
  78. zenml/zen_server/dashboard/assets/{404-C1mcUujL.js → 404-DpJaNHKF.js} +1 -1
  79. zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +85 -0
  80. zenml/zen_server/dashboard/assets/{@react-router-DYovave8.js → @react-router-CO-OsFwI.js} +2 -2
  81. zenml/zen_server/dashboard/assets/{@reactflow-DYIyhCfd.js → @reactflow-DJfzkHO1.js} +2 -2
  82. zenml/zen_server/dashboard/assets/@tanstack-DYiOyJUL.js +22 -0
  83. zenml/zen_server/dashboard/assets/AwarenessChannel-BYDLT2xC.js +1 -0
  84. zenml/zen_server/dashboard/assets/{CodeSnippet-WEzpO0az.js → CodeSnippet-BkOuRmyq.js} +2 -2
  85. zenml/zen_server/dashboard/assets/Commands-ZvWR1BRs.js +1 -0
  86. zenml/zen_server/dashboard/assets/CopyButton-DVwLkafa.js +2 -0
  87. zenml/zen_server/dashboard/assets/{CsvVizualization-Bx931j4U.js → CsvVizualization-C2IiqX4I.js} +7 -7
  88. zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +1 -0
  89. zenml/zen_server/dashboard/assets/EmptyState-BMLnFVlB.js +1 -0
  90. zenml/zen_server/dashboard/assets/Error-CqX0VqW_.js +1 -0
  91. zenml/zen_server/dashboard/assets/ExecutionStatus-BoLUXR9t.js +1 -0
  92. zenml/zen_server/dashboard/assets/Helpbox-LFydyVwh.js +1 -0
  93. zenml/zen_server/dashboard/assets/Infobox-DnENC0sh.js +1 -0
  94. zenml/zen_server/dashboard/assets/InlineAvatar-CbJtYr0t.js +1 -0
  95. zenml/zen_server/dashboard/assets/{MarkdownVisualization-DsB2QZiK.js → MarkdownVisualization-xp3hhULl.js} +2 -2
  96. zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +1 -0
  97. zenml/zen_server/dashboard/assets/PasswordChecker-DUveqlva.js +1 -0
  98. zenml/zen_server/dashboard/assets/SetPassword-BYBdbQDo.js +1 -0
  99. zenml/zen_server/dashboard/assets/SuccessStep-Nx743hll.js +1 -0
  100. zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CKrd3UZz.js → UpdatePasswordSchemas-DF9gSzE0.js} +1 -1
  101. zenml/zen_server/dashboard/assets/{aws-t0gKCj_R.js → aws-BgKTfTfx.js} +1 -1
  102. zenml/zen_server/dashboard/assets/{check-circle-BVvhm5dy.js → check-circle-i56092KI.js} +1 -1
  103. zenml/zen_server/dashboard/assets/{chevron-down-zcvCWmyP.js → chevron-down-D_ZlKMqH.js} +1 -1
  104. zenml/zen_server/dashboard/assets/{chevron-right-double-CJ50E9Gr.js → chevron-right-double-BiEMg7rd.js} +1 -1
  105. zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +1 -0
  106. zenml/zen_server/dashboard/assets/{copy-BRhQz3j-.js → copy-BXNk6BjL.js} +1 -1
  107. zenml/zen_server/dashboard/assets/{database-CRRnyFWh.js → database-1xWSgZfO.js} +1 -1
  108. zenml/zen_server/dashboard/assets/{docker-BAonhm6G.js → docker-CQMVm_4d.js} +1 -1
  109. zenml/zen_server/dashboard/assets/{file-text-CbVERUON.js → file-text-CqD_iu6l.js} +1 -1
  110. zenml/zen_server/dashboard/assets/{help-B8rqCvqn.js → help-bu_DgLKI.js} +1 -1
  111. zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +1 -0
  112. zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +55 -0
  113. zenml/zen_server/dashboard/assets/index-inApY3KQ.css +1 -0
  114. zenml/zen_server/dashboard/assets/index-rK_Wuy2W.js +1 -0
  115. zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +1 -0
  116. zenml/zen_server/dashboard/assets/{login-mutation-Bk2tn324.js → login-mutation-BUnVASxp.js} +1 -1
  117. zenml/zen_server/dashboard/assets/not-found-B4VnX8gK.js +1 -0
  118. zenml/zen_server/dashboard/assets/package-CsUhPmou.js +1 -0
  119. zenml/zen_server/dashboard/assets/{page-D12Rvf0j.js → page-3efNCDeb.js} +2 -2
  120. zenml/zen_server/dashboard/assets/page-7zTHbhhI.js +1 -0
  121. zenml/zen_server/dashboard/assets/page-BEs6jK71.js +1 -0
  122. zenml/zen_server/dashboard/assets/page-BpSqIf4B.js +1 -0
  123. zenml/zen_server/dashboard/assets/{page-8vRWJ5b8.js → page-Bx6o0ARS.js} +1 -1
  124. zenml/zen_server/dashboard/assets/page-C43QGHTt.js +9 -0
  125. zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +1 -0
  126. zenml/zen_server/dashboard/assets/{page-CBuSUrE9.js → page-CRTJ0UuR.js} +1 -1
  127. zenml/zen_server/dashboard/assets/page-CUZIGO-3.js +1 -0
  128. zenml/zen_server/dashboard/assets/page-CaopxiU1.js +1 -0
  129. zenml/zen_server/dashboard/assets/{page-CCtCgG-x.js → page-Cx67M0QT.js} +1 -1
  130. zenml/zen_server/dashboard/assets/page-D7Z399xy.js +1 -0
  131. zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +1 -0
  132. zenml/zen_server/dashboard/assets/{page-Dw9-aJV6.js → page-DKlIdAe5.js} +1 -1
  133. zenml/zen_server/dashboard/assets/{page-COafKNbw.js → page-DMOYZppS.js} +1 -1
  134. zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +2 -0
  135. zenml/zen_server/dashboard/assets/{page-C6v3o0Qj.js → page-Dc_7KMQE.js} +1 -1
  136. zenml/zen_server/dashboard/assets/page-DvCvroOM.js +1 -0
  137. zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +1 -0
  138. zenml/zen_server/dashboard/assets/page-JyfeDUfu.js +1 -0
  139. zenml/zen_server/dashboard/assets/{page-CH26py0a.js → page-Sxn82W-5.js} +1 -1
  140. zenml/zen_server/dashboard/assets/page-TKXERe16.js +1 -0
  141. zenml/zen_server/dashboard/assets/page-Xu8JEjSU.js +1 -0
  142. zenml/zen_server/dashboard/assets/{play-circle-DK5QMJyp.js → play-circle-CNtZKDnW.js} +1 -1
  143. zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +1 -0
  144. zenml/zen_server/dashboard/assets/{terminal-B2ovgWuz.js → terminal-By9cErXc.js} +1 -1
  145. zenml/zen_server/dashboard/assets/{update-server-settings-mutation-bKxf7U9h.js → update-server-settings-mutation-CR8e3Sir.js} +1 -1
  146. zenml/zen_server/dashboard/assets/{url-CgvM-IVM.js → url-DuQMeqYA.js} +1 -1
  147. zenml/zen_server/dashboard/assets/{zod-DrZvVLjd.js → zod-BhoGpZ63.js} +1 -1
  148. zenml/zen_server/dashboard/index.html +7 -7
  149. zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
  150. zenml/zen_server/dashboard_legacy/index.html +1 -1
  151. zenml/zen_server/dashboard_legacy/{precache-manifest.e7c29295aae591541ef59d1734d79387.js → precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js} +4 -4
  152. zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
  153. zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js → main.382439a7.chunk.js} +2 -2
  154. zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js.map → main.382439a7.chunk.js.map} +1 -1
  155. zenml/zen_server/deploy/helm/Chart.yaml +1 -1
  156. zenml/zen_server/deploy/helm/README.md +2 -2
  157. zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +11 -5
  158. zenml/zen_server/pipeline_deployment/utils.py +57 -44
  159. zenml/zen_server/rbac/zenml_cloud_rbac.py +11 -5
  160. zenml/zen_server/routers/stack_deployment_endpoints.py +144 -0
  161. zenml/zen_server/routers/workspaces_endpoints.py +64 -0
  162. zenml/zen_server/zen_server_api.py +2 -0
  163. zenml/zen_stores/migrations/utils.py +1 -1
  164. zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
  165. zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
  166. zenml/zen_stores/rest_zen_store.py +117 -0
  167. zenml/zen_stores/schemas/stack_schemas.py +10 -0
  168. zenml/zen_stores/schemas/step_run_schemas.py +27 -11
  169. zenml/zen_stores/sql_zen_store.py +283 -0
  170. zenml/zen_stores/zen_store_interface.py +79 -0
  171. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/METADATA +32 -10
  172. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/RECORD +175 -161
  173. zenml/zen_server/dashboard/assets/@radix-C9DBgJhe.js +0 -77
  174. zenml/zen_server/dashboard/assets/@tanstack-CEbkxrhX.js +0 -30
  175. zenml/zen_server/dashboard/assets/AwarenessChannel-B2KR83Tr.js +0 -1
  176. zenml/zen_server/dashboard/assets/Cards-DSEdjsk8.js +0 -1
  177. zenml/zen_server/dashboard/assets/Commands-CTlhyic5.js +0 -1
  178. zenml/zen_server/dashboard/assets/CopyButton-CTrzKmUO.js +0 -2
  179. zenml/zen_server/dashboard/assets/DisplayDate-BdguISQF.js +0 -1
  180. zenml/zen_server/dashboard/assets/EmptyState-BkooiGtL.js +0 -1
  181. zenml/zen_server/dashboard/assets/Error-4sKxHad4.js +0 -1
  182. zenml/zen_server/dashboard/assets/Helpbox-DW21i5LD.js +0 -1
  183. zenml/zen_server/dashboard/assets/Infobox-C7bf70VS.js +0 -1
  184. zenml/zen_server/dashboard/assets/InlineAvatar-Dxrtafpg.js +0 -1
  185. zenml/zen_server/dashboard/assets/PageHeader-B0pUife2.js +0 -1
  186. zenml/zen_server/dashboard/assets/Pagination-B9WG_9cJ.js +0 -1
  187. zenml/zen_server/dashboard/assets/PasswordChecker-DSLBp7Vl.js +0 -1
  188. zenml/zen_server/dashboard/assets/SetPassword-CiNhT15a.js +0 -1
  189. zenml/zen_server/dashboard/assets/SuccessStep-CykrFndS.js +0 -1
  190. zenml/zen_server/dashboard/assets/cloud-only-Bkawp7CJ.js +0 -1
  191. zenml/zen_server/dashboard/assets/index-BawkpTlr.js +0 -55
  192. zenml/zen_server/dashboard/assets/index-CRmm7QhS.css +0 -1
  193. zenml/zen_server/dashboard/assets/index.esm-F7nqy9zY.js +0 -1
  194. zenml/zen_server/dashboard/assets/not-found-BAuhP4Jb.js +0 -1
  195. zenml/zen_server/dashboard/assets/page--5YvAHg3.js +0 -1
  196. zenml/zen_server/dashboard/assets/page-B0RAq4s_.js +0 -1
  197. zenml/zen_server/dashboard/assets/page-BePtEPHl.js +0 -1
  198. zenml/zen_server/dashboard/assets/page-C1pra1Bc.js +0 -9
  199. zenml/zen_server/dashboard/assets/page-CSs4C9jL.js +0 -1
  200. zenml/zen_server/dashboard/assets/page-Cf2XSej0.js +0 -1
  201. zenml/zen_server/dashboard/assets/page-ClPUAE_f.js +0 -1
  202. zenml/zen_server/dashboard/assets/page-D8pf2vis.js +0 -1
  203. zenml/zen_server/dashboard/assets/page-DHKMmIQH.js +0 -1
  204. zenml/zen_server/dashboard/assets/page-DMZ0VOda.js +0 -1
  205. zenml/zen_server/dashboard/assets/page-Dcg-yQv_.js +0 -1
  206. zenml/zen_server/dashboard/assets/page-DoAK5FSB.js +0 -1
  207. zenml/zen_server/dashboard/assets/page-iXiDqE0J.js +0 -2
  208. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/LICENSE +0 -0
  209. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/WHEEL +0 -0
  210. {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/entry_points.txt +0 -0
@@ -43,7 +43,8 @@ from google.auth._default import (
43
43
  _get_external_account_credentials,
44
44
  )
45
45
  from google.auth.transport.requests import Request
46
- from google.cloud import container_v1, storage
46
+ from google.cloud import artifactregistry_v1, container_v1, storage
47
+ from google.cloud.location import locations_pb2
47
48
  from google.oauth2 import credentials as gcp_credentials
48
49
  from google.oauth2 import service_account as gcp_service_account
49
50
  from pydantic import Field, field_validator, model_validator
@@ -834,14 +835,53 @@ GKE clusters in the GCP project that it is configured to use.
834
835
  emoji=":cyclone:",
835
836
  ),
836
837
  ResourceTypeModel(
837
- name="GCP GCR container registry",
838
+ name="GCP GAR container registry",
838
839
  resource_type=DOCKER_REGISTRY_RESOURCE_TYPE,
839
840
  description="""
840
- Allows Stack Components to access a GCR registry as a standard
841
+ Allows Stack Components to access a Google Artifact Registry as a standard
841
842
  Docker registry resource. When used by Stack Components, they are provided a
842
843
  pre-authenticated Python Docker client instance.
843
844
 
844
- The configured credentials must have at least the following [GCP permissions](https://cloud.google.com/iam/docs/permissions-reference):
845
+ The configured credentials must have at least the following [GCP permissions](https://cloud.google.com/iam/docs/understanding-roles#artifact-registry-roles):
846
+
847
+ - `artifactregistry.repositories.createOnPush`
848
+ - `artifactregistry.repositories.downloadArtifacts`
849
+ - `artifactregistry.repositories.get`
850
+ - `artifactregistry.repositories.list`
851
+ - `artifactregistry.repositories.readViaVirtualRepository`
852
+ - `artifactregistry.repositories.uploadArtifacts`
853
+ - `artifactregistry.locations.list`
854
+
855
+ The Artifact Registry Create-on-Push Writer role includes all of the above
856
+ permissions.
857
+
858
+ This resource type also includes legacy GCR container registry support.
859
+
860
+ **Important Notice: Google Container Registry** [**is being replaced by Artifact Registry**](https://cloud.google.com/artifact-registry/docs/transition/transition-from-gcr).
861
+ Please start using Artifact Registry for your containers. As per Google's
862
+ documentation, *"after May 15, 2024, Artifact Registry will host images for the
863
+ gcr.io domain in Google Cloud projects without previous Container Registry
864
+ usage. After March 18, 2025, Container Registry will be shut down."*.
865
+
866
+ Support for legacy GCR registries is still included in the GCP service
867
+ connector. Users that already have GCP service connectors configured to access
868
+ GCR registries may continue to use them without taking any action. However, it
869
+ is recommended to transition to Google Artifact Registries as soon as possible
870
+ by following [the GCP guide on this subject](https://cloud.google.com/artifact-registry/docs/transition/transition-from-gcr)
871
+ and making the following updates to ZenML GCP Service Connectors that are used
872
+ to access GCR resources:
873
+
874
+ * add the IAM permissions documented here to the GCP Service Connector
875
+ credentials to enable them to access the Artifact Registries.
876
+ * users may keep the gcr.io GCR URLs already configured in the GCP Service
877
+ Connectors as well as those used in linked Container Registry stack components
878
+ given that these domains are redirected by Google to GAR as covered in the GCR
879
+ transition guide. Alternatively, users may update the GCP Service Connector
880
+ configuration and/or the Container Registry stack components to use the
881
+ replacement Artifact Registry URLs.
882
+
883
+ When used with GCR registries, the configured credentials must have at least the
884
+ following [GCP permissions](https://cloud.google.com/iam/docs/understanding-roles#cloud-storage-roles):
845
885
 
846
886
  - `storage.buckets.get`
847
887
  - `storage.multipartUploads.abort`
@@ -855,17 +895,21 @@ The configured credentials must have at least the following [GCP permissions](ht
855
895
  The Storage Legacy Bucket Writer role includes all of the above permissions
856
896
  while at the same time restricting access to only the GCR buckets.
857
897
 
858
- The resource name associated with this resource type identifies the GCR
859
- container registry associated with the GCP project (the repository name is
860
- optional):
898
+ If set, the resource name must identify a GAR or GCR registry using one of the
899
+ following formats:
900
+
901
+ - Google Artifact Registry repository URI: `[https://]<region>-docker.pkg.dev/<project-id>/<registry-id>[/<repository-name>]`
902
+ - Google Artifact Registry name: `projects/<project-id>/locations/<location>/repositories/<repository-id>`
903
+ - (legacy) GCR repository URI: `[https://][us.|eu.|asia.]gcr.io/<project-id>[/<repository-name>]`
861
904
 
862
- - GCR repository URI: `[https://]gcr.io/{project-id}[/{repository-name}]
905
+ The connector can only be used to access GAR and GCR registries in the GCP
906
+ project that it is configured to use.
863
907
  """,
864
908
  auth_methods=GCPAuthenticationMethods.values(),
865
- # Does not support instances, given that the connector
866
- # provides access to the entire GCR container registry
867
- # for the configured GCP project.
868
- supports_instances=False,
909
+ # The connector provides access to the entire GCR container registry
910
+ # for the configured GCP project as well as any number of artifact
911
+ # registry repositories.
912
+ supports_instances=True,
869
913
  logo_url="https://public-flavor-logos.s3.eu-central-1.amazonaws.com/container_registry/docker.png",
870
914
  emoji=":whale:",
871
915
  ),
@@ -1142,60 +1186,101 @@ class GCPServiceConnector(ServiceConnector):
1142
1186
 
1143
1187
  return bucket_name
1144
1188
 
1145
- def _parse_gcr_resource_id(
1189
+ def _parse_gar_resource_id(
1146
1190
  self,
1147
1191
  resource_id: str,
1148
- ) -> str:
1149
- """Validate and convert an GCR resource ID to an GCR registry ID.
1192
+ ) -> Tuple[str, Optional[str]]:
1193
+ """Validate and convert a GAR resource ID to a Google Artifact Registry ID and name.
1150
1194
 
1151
1195
  Args:
1152
1196
  resource_id: The resource ID to convert.
1153
1197
 
1154
1198
  Returns:
1155
- The GCR registry ID.
1199
+ The Google Artifact Registry ID and name. The name is omitted if the
1200
+ resource ID is a GCR repository URI.
1156
1201
 
1157
1202
  Raises:
1158
- ValueError: If the provided resource ID is not a valid GCR
1159
- repository URI.
1203
+ ValueError: If the provided resource ID is not a valid GAR
1204
+ or GCR repository URI.
1160
1205
  """
1161
1206
  # The resource ID could mean different things:
1162
1207
  #
1163
- # - an GCR repository URI
1208
+ # - a GAR repository URI
1209
+ # - a GAR repository name
1210
+ # - a GCR repository URI (backwards-compatibility)
1164
1211
  #
1165
1212
  # We need to extract the project ID and registry ID from
1166
1213
  # the provided resource ID
1167
1214
  config_project_id = self.config.gcp_project_id
1168
1215
  project_id: Optional[str] = None
1169
- # A GCR repository URI uses one of several hostnames (gcr.io, us.gcr.io,
1170
- # eu.gcr.io, asia.gcr.io etc.) and the project ID is the first part of
1171
- # the URL path
1172
- if re.match(
1173
- r"^(https://)?([a-z]+.)*gcr.io/[a-z0-9-]+(/.+)*$",
1216
+ canonical_url: str
1217
+ registry_name: Optional[str] = None
1218
+
1219
+ # A Google Artifact Registry URI uses the <location>-docker-pkg.dev
1220
+ # domain format with the project ID as the first part of the URL path
1221
+ # and the registry name as the second part of the URL path
1222
+ if match := re.match(
1223
+ r"^(https://)?(([a-z0-9-]+)-docker\.pkg\.dev/([a-z0-9-]+)/([a-z0-9-.]+))(/.+)*$",
1174
1224
  resource_id,
1175
1225
  ):
1176
- # The resource ID is a GCR repository URI
1177
- if resource_id.startswith("https://"):
1178
- project_id = resource_id.split("/")[3]
1179
- else:
1180
- project_id = resource_id.split("/")[1]
1226
+ # The resource ID is a Google Artifact Registry URI
1227
+ project_id = match[4]
1228
+ location = match[3]
1229
+ repository = match[5]
1230
+
1231
+ # Return the GAR URL without the image name and without the protocol
1232
+ canonical_url = match[2]
1233
+ registry_name = f"projects/{project_id}/locations/{location}/repositories/{repository}"
1234
+
1235
+ # Alternatively, the Google Artifact Registry name uses the
1236
+ # projects/<project-id>/locations/<location>/repositories/<repository-id>
1237
+ # format
1238
+ elif match := re.match(
1239
+ r"^projects/([a-z0-9-]+)/locations/([a-z0-9-]+)/repositories/([a-z0-9-.]+)$",
1240
+ resource_id,
1241
+ ):
1242
+ # The resource ID is a Google Artifact Registry name
1243
+ project_id = match[1]
1244
+ location = match[2]
1245
+ repository = match[3]
1246
+
1247
+ # Return the GAR URL
1248
+ canonical_url = (
1249
+ f"{location}-docker.pkg.dev/{project_id}/{repository}"
1250
+ )
1251
+ registry_name = resource_id
1252
+
1253
+ # A legacy GCR repository URI uses one of several hostnames (gcr.io,
1254
+ # us.gcr.io, eu.gcr.io, asia.gcr.io) and the project ID is the
1255
+ # first part of the URL path
1256
+ elif match := re.match(
1257
+ r"^(https://)?(((us|eu|asia)\.)?gcr\.io/[a-z0-9-]+)(/.+)*$",
1258
+ resource_id,
1259
+ ):
1260
+ # The resource ID is a legacy GCR repository URI.
1261
+ # Return the GAR URL without the image name and without the protocol
1262
+ canonical_url = match[2]
1263
+
1181
1264
  else:
1182
1265
  raise ValueError(
1183
- f"Invalid resource ID for a GCR registry: {resource_id}. "
1184
- f"Supported formats are:\n"
1266
+ f"Invalid resource ID for a Google Artifact Registry: "
1267
+ f"{resource_id}. Supported formats are:\n"
1268
+ f"Google Artifact Registry URI: [https://]<region>-docker.pkg.dev/<project-id>/<registry-id>[/<repository-name>]\n"
1269
+ f"Google Artifact Registry name: projects/<project-id>/locations/<location>/repositories/<repository-id>\n"
1185
1270
  f"GCR repository URI: [https://][us.|eu.|asia.]gcr.io/<project-id>[/<repository-name>]"
1186
1271
  )
1187
1272
 
1188
1273
  # If the connector is configured with a project and the resource ID
1189
- # is an GCR repository URI that specifies a different project,
1274
+ # is a GAR repository URI that specifies a different project,
1190
1275
  # we raise an error
1191
1276
  if project_id and project_id != config_project_id:
1192
1277
  raise ValueError(
1193
- f"The GCP project for the {resource_id} GCR repository "
1194
- f"'{project_id}' does not match the project configured in "
1195
- f"the connector: '{config_project_id}'."
1278
+ f"The GCP project for the {resource_id} Google Artifact "
1279
+ f"Registry '{project_id}' does not match the project "
1280
+ f"configured in the connector: '{config_project_id}'."
1196
1281
  )
1197
1282
 
1198
- return f"gcr.io/{project_id}"
1283
+ return canonical_url, registry_name
1199
1284
 
1200
1285
  def _parse_gke_resource_id(self, resource_id: str) -> str:
1201
1286
  """Validate and convert an GKE resource ID to a GKE cluster name.
@@ -1244,7 +1329,7 @@ class GCPServiceConnector(ServiceConnector):
1244
1329
  cluster_name = self._parse_gke_resource_id(resource_id)
1245
1330
  return cluster_name
1246
1331
  elif resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
1247
- registry_id = self._parse_gcr_resource_id(
1332
+ registry_id, _ = self._parse_gar_resource_id(
1248
1333
  resource_id,
1249
1334
  )
1250
1335
  return registry_id
@@ -1269,8 +1354,6 @@ class GCPServiceConnector(ServiceConnector):
1269
1354
  """
1270
1355
  if resource_type == GCP_RESOURCE_TYPE:
1271
1356
  return self.config.gcp_project_id
1272
- elif resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
1273
- return f"gcr.io/{self.config.gcp_project_id}"
1274
1357
 
1275
1358
  raise RuntimeError(
1276
1359
  f"Default resource ID not supported for '{resource_type}' resource "
@@ -1771,11 +1854,87 @@ class GCPServiceConnector(ServiceConnector):
1771
1854
  raise AuthorizationException(msg) from e
1772
1855
 
1773
1856
  if resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
1774
- assert resource_id is not None
1857
+ # Get a GAR client
1858
+ gar_client = artifactregistry_v1.ArtifactRegistryClient(
1859
+ credentials=credentials
1860
+ )
1775
1861
 
1776
- # No way to verify a GCR registry without attempting to
1777
- # connect to it via Docker/OCI, so just return the resource ID.
1778
- return [resource_id]
1862
+ if resource_id:
1863
+ registry_id, registry_name = self._parse_gar_resource_id(
1864
+ resource_id
1865
+ )
1866
+
1867
+ if registry_name is None:
1868
+ # This is a legacy GCR repository URI. We can't verify
1869
+ # the repository access without attempting to connect to it
1870
+ # via Docker/OCI, so just return the resource ID.
1871
+ return [registry_id]
1872
+
1873
+ # Check if the specified GAR registry exists
1874
+ try:
1875
+ repository = gar_client.get_repository(
1876
+ name=registry_name,
1877
+ )
1878
+ if repository.format_.name != "DOCKER":
1879
+ raise AuthorizationException(
1880
+ f"Google Artifact Registry '{resource_id}' is not a "
1881
+ "Docker registry."
1882
+ )
1883
+ return [registry_id]
1884
+ except google.api_core.exceptions.GoogleAPIError as e:
1885
+ msg = f"Failed to fetch Google Artifact Registry '{registry_id}': {e}"
1886
+ logger.error(msg)
1887
+ raise AuthorizationException(msg) from e
1888
+
1889
+ # For backwards compatibility, we initialize the list of resource
1890
+ # IDs with all GCR supported registries for the configured GCP
1891
+ # project
1892
+ resource_ids: List[str] = [
1893
+ f"{location}gcr.io/{self.config.gcp_project_id}"
1894
+ for location in ["", "us.", "eu.", "asia."]
1895
+ ]
1896
+
1897
+ # List all Google Artifact Registries
1898
+ try:
1899
+ # First, we need to fetch all the Artifact Registry supported
1900
+ # locations
1901
+ locations = gar_client.list_locations(
1902
+ request=locations_pb2.ListLocationsRequest(
1903
+ name=f"projects/{self.config.gcp_project_id}"
1904
+ )
1905
+ )
1906
+ location_names = [
1907
+ locations.locations[i].location_id
1908
+ for i in range(len(locations.locations))
1909
+ ]
1910
+
1911
+ # Then, we need to fetch all the repositories in each location
1912
+ repository_names: List[str] = []
1913
+ for location in location_names:
1914
+ repositories = gar_client.list_repositories(
1915
+ parent=f"projects/{self.config.gcp_project_id}/locations/{location}"
1916
+ )
1917
+ repository_names.extend(
1918
+ [
1919
+ repository.name
1920
+ for repository in repositories
1921
+ if repository.format_.name == "DOCKER"
1922
+ ]
1923
+ )
1924
+
1925
+ for repository_name in repository_names:
1926
+ # Convert the repository name to a canonical GAR URL
1927
+ resource_ids.append(
1928
+ self._parse_gar_resource_id(repository_name)[0]
1929
+ )
1930
+
1931
+ except google.api_core.exceptions.GoogleAPIError as e:
1932
+ msg = f"Failed to list Google Artifact Registries: {e}"
1933
+ logger.error(msg)
1934
+ # TODO: enable when GCR is no longer supported:
1935
+ # raise AuthorizationException(msg) from e
1936
+
1937
+ return resource_ids
1779
1938
 
1780
1939
  if resource_type == KUBERNETES_CLUSTER_RESOURCE_TYPE:
1781
1940
  gke_client = container_v1.ClusterManagerClient(
@@ -1904,7 +2063,7 @@ class GCPServiceConnector(ServiceConnector):
1904
2063
  if resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
1905
2064
  assert resource_id is not None
1906
2065
 
1907
- registry_id = self._parse_gcr_resource_id(resource_id)
2066
+ registry_id, _ = self._parse_gar_resource_id(resource_id)
1908
2067
 
1909
2068
  # Create a client-side Docker connector instance with the temporary
1910
2069
  # Docker credentials
@@ -39,6 +39,7 @@ class HuggingfaceIntegration(Integration):
39
39
  # TODO try relaxing it back going forward
40
40
  "fsspec<=2023.12.0",
41
41
  ]
42
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["fsspec"]
42
43
 
43
44
  @classmethod
44
45
  def activate(cls) -> None:
@@ -60,6 +60,7 @@ class Integration(metaclass=IntegrationMeta):
60
60
 
61
61
  REQUIREMENTS: List[str] = []
62
62
  APT_PACKAGES: List[str] = []
63
+ REQUIREMENTS_IGNORED_ON_UNINSTALL: List[str] = []
63
64
 
64
65
  @classmethod
65
66
  def check_installation(cls) -> bool:
@@ -143,6 +144,29 @@ class Integration(metaclass=IntegrationMeta):
143
144
  """
144
145
  return cls.REQUIREMENTS
145
146
 
147
+ @classmethod
148
+ def get_uninstall_requirements(
149
+ cls, target_os: Optional[str] = None
150
+ ) -> List[str]:
151
+ """Method to get the uninstall requirements for the integration.
152
+
153
+ Args:
154
+ target_os: The target operating system to get the requirements for.
155
+
156
+ Returns:
157
+ A list of requirements.
158
+ """
159
+ ret = []
160
+ for each in cls.get_requirements(target_os=target_os):
161
+ is_ignored = False
162
+ for ignored in cls.REQUIREMENTS_IGNORED_ON_UNINSTALL:
163
+ if each.startswith(ignored):
164
+ is_ignored = True
165
+ break
166
+ if not is_ignored:
167
+ ret.append(each)
168
+ return ret
169
+
146
170
  @classmethod
147
171
  def activate(cls) -> None:
148
172
  """Abstract method to activate the integration."""
@@ -31,6 +31,9 @@ class KubeflowIntegration(Integration):
31
31
 
32
32
  NAME = KUBEFLOW
33
33
  REQUIREMENTS = ["kfp>=2.6.0", "kfp-kubernetes>=1.1.0"] # Only 1.x version that supports pyyaml 6
34
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = [
35
+ "kfp", # it is used by GCP as well
36
+ ]
34
37
 
35
38
  @classmethod
36
39
  def flavors(cls) -> List[Type[Flavor]]:
@@ -149,7 +149,7 @@ class KubeflowOrchestratorConfig(
149
149
 
150
150
  kubeflow_hostname: Optional[str] = None
151
151
  kubeflow_namespace: str = "kubeflow"
152
- kubernetes_context: Optional[str] # TODO: Potential setting
152
+ kubernetes_context: Optional[str] = None # TODO: Potential setting
153
153
 
154
154
  @model_validator(mode="before")
155
155
  @classmethod
@@ -433,7 +433,6 @@ class KubeflowOrchestrator(ContainerizedOrchestrator):
433
433
  The dynamic container component.
434
434
  """
435
435
 
436
- @dsl.container_component # type: ignore[misc]
437
436
  def dynamic_container_component() -> dsl.ContainerSpec:
438
437
  """Dynamic container component.
439
438
 
@@ -31,7 +31,9 @@ class KubernetesIntegration(Integration):
31
31
 
32
32
  NAME = KUBERNETES
33
33
  REQUIREMENTS = ["kubernetes>=21.7,<26"]
34
-
34
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = [
35
+ "kfp", # it is used by many others
36
+ ]
35
37
  @classmethod
36
38
  def flavors(cls) -> List[Type[Flavor]]:
37
39
  """Declare the stack component flavors for the Kubernetes integration.
@@ -225,8 +225,11 @@ def wait_pod(
225
225
  response = core_api.read_namespaced_pod_log(
226
226
  name=pod_name,
227
227
  namespace=namespace,
228
+ _preload_content=False,
228
229
  )
229
- logs = response.splitlines()
230
+ raw_data = response.data
231
+ decoded_log = raw_data.decode("utf-8", errors="replace")
232
+ logs = decoded_log.splitlines()
230
233
  if len(logs) > logged_lines:
231
234
  for line in logs[logged_lines:]:
232
235
  logger.info(line)
@@ -527,6 +527,7 @@ class LabelStudioAnnotator(BaseAnnotator, AuthenticationMixin):
527
527
  aws_access_key_id,
528
528
  aws_secret_access_key,
529
529
  aws_session_token,
530
+ _,
530
531
  ) = artifact_store.get_credentials()
531
532
 
532
533
  if aws_access_key_id and aws_secret_access_key:
@@ -30,6 +30,7 @@ class LangchainIntegration(Integration):
30
30
  "pyyaml>=6.0.1",
31
31
  "tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
32
32
  ]
33
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pyyaml","tenacity"]
33
34
 
34
35
  @classmethod
35
36
  def activate(cls) -> None:
@@ -45,9 +45,11 @@ class MlflowIntegration(Integration):
45
45
  # This downgrades pydantic to v1 even though mlflow does not have
46
46
  # any issues with v2. This is why we have to pin it here so a downgrade
47
47
  # will not happen.
48
- "pydantic>=2.7.0,<2.8.0"
48
+ "pydantic>=2.7.0,<2.8.0",
49
49
  ]
50
50
 
51
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["python-rapidjson", "pydantic"]
52
+
51
53
  @classmethod
52
54
  def activate(cls) -> None:
53
55
  """Activate the MLflow integration."""
@@ -26,6 +26,7 @@ class NeuralProphetIntegration(Integration):
26
26
  "holidays>=0.4.1,<0.25.0",
27
27
  "tenacity!=8.4.0", # https://github.com/jd/tenacity/issues/471
28
28
  ]
29
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]
29
30
 
30
31
  @classmethod
31
32
  def activate(cls) -> None:
@@ -25,6 +25,7 @@ class PolarsIntegration(Integration):
25
25
  "polars>=0.19.5",
26
26
  "pyarrow>=12.0.0"
27
27
  ]
28
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pyarrow"]
28
29
 
29
30
  @classmethod
30
31
  def activate(cls) -> None:
@@ -29,6 +29,7 @@ class ProdigyIntegration(Integration):
29
29
  "prodigy",
30
30
  "urllib3<2",
31
31
  ]
32
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["urllib3"]
32
33
 
33
34
  @classmethod
34
35
  def flavors(cls) -> List[Type[Flavor]]:
@@ -28,6 +28,12 @@ class PyCaretIntegration(Integration):
28
28
  "catboost",
29
29
  "lightgbm",
30
30
  ]
31
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = [
32
+ "scikit-learn",
33
+ "xgboost",
34
+ "catboost",
35
+ "lightgbm",
36
+ ]
31
37
 
32
38
  @classmethod
33
39
  def activate(cls) -> None:
@@ -124,6 +124,43 @@ class IntegrationRegistry(object):
124
124
  )
125
125
  ]
126
126
 
127
+ def select_uninstall_requirements(
128
+ self,
129
+ integration_name: Optional[str] = None,
130
+ target_os: Optional[str] = None,
131
+ ) -> List[str]:
132
+ """Select the uninstall requirements for a given integration or all integrations.
133
+
134
+ Args:
135
+ integration_name: Name of the integration to check.
136
+ target_os: Target OS for the requirements.
137
+
138
+ Returns:
139
+ List of requirements for the integration uninstall.
140
+
141
+ Raises:
142
+ KeyError: If the integration is not found.
143
+ """
144
+ if integration_name:
145
+ if integration_name in self.list_integration_names:
146
+ return self._integrations[
147
+ integration_name
148
+ ].get_uninstall_requirements(target_os=target_os)
149
+ else:
150
+ raise KeyError(
151
+ f"Version {integration_name} does not exist. "
152
+ f"Currently the following integrations are implemented. "
153
+ f"{self.list_integration_names}"
154
+ )
155
+ else:
156
+ return [
157
+ requirement
158
+ for name in self.list_integration_names
159
+ for requirement in self._integrations[
160
+ name
161
+ ].get_uninstall_requirements(target_os=target_os)
162
+ ]
163
+
127
164
  def is_installed(self, integration_name: Optional[str] = None) -> bool:
128
165
  """Checks if all requirements for an integration are installed.
129
166
 
@@ -122,7 +122,7 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
122
122
 
123
123
  def get_credentials(
124
124
  self,
125
- ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
125
+ ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
126
126
  """Gets authentication credentials.
127
127
 
128
128
  If an authentication secret is configured, the secret values are
@@ -130,8 +130,8 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
130
130
  attributes.
131
131
 
132
132
  Returns:
133
- Tuple (key, secret, token) of credentials used to authenticate with
134
- the S3 filesystem.
133
+ Tuple (key, secret, token, region) of credentials used to
134
+ authenticate with the S3 filesystem.
135
135
 
136
136
  Raises:
137
137
  RuntimeError: If the AWS connector behaves unexpectedly.
@@ -151,6 +151,7 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
151
151
  credentials.access_key,
152
152
  credentials.secret_key,
153
153
  credentials.token,
154
+ client.meta.region_name,
154
155
  )
155
156
 
156
157
  secret = self.get_typed_authentication_secret(
@@ -161,9 +162,10 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
161
162
  secret.aws_access_key_id,
162
163
  secret.aws_secret_access_key,
163
164
  secret.aws_session_token,
165
+ None,
164
166
  )
165
167
  else:
166
- return self.config.key, self.config.secret, self.config.token
168
+ return self.config.key, self.config.secret, self.config.token, None
167
169
 
168
170
  @property
169
171
  def filesystem(self) -> ZenMLS3Filesystem:
@@ -176,13 +178,22 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
176
178
  if self._filesystem and not self.connector_has_expired():
177
179
  return self._filesystem
178
180
 
179
- key, secret, token = self.get_credentials()
181
+ key, secret, token, region = self.get_credentials()
182
+
183
+ # Use the region from the connector if available, otherwise some
184
+ # remote workloads (e.g. Sagemaker) might not work correctly because
185
+ # they look for the bucket in the wrong region
186
+ client_kwargs = {}
187
+ if region:
188
+ client_kwargs["region_name"] = region
189
+ if self.config.client_kwargs:
190
+ client_kwargs.update(self.config.client_kwargs)
180
191
 
181
192
  self._filesystem = ZenMLS3Filesystem(
182
193
  key=key,
183
194
  secret=secret,
184
195
  token=token,
185
- client_kwargs=self.config.client_kwargs,
196
+ client_kwargs=client_kwargs,
186
197
  config_kwargs=self.config.config_kwargs,
187
198
  s3_additional_kwargs=self.config.s3_additional_kwargs,
188
199
  )
@@ -33,6 +33,7 @@ class SeldonIntegration(Integration):
33
33
  REQUIREMENTS = [
34
34
  "kubernetes==18.20.0",
35
35
  ]
36
+ REQUIREMENTS_IGNORED_ON_UNINSTALL = ["kubernetes"]
36
37
 
37
38
  @classmethod
38
39
  def activate(cls) -> None:
@@ -290,6 +290,7 @@ class SeldonModelDeployer(BaseModelDeployer):
290
290
  assert isinstance(artifact_store, S3ArtifactStore)
291
291
 
292
292
  (
293
+ region_name,
293
294
  aws_access_key_id,
294
295
  aws_secret_access_key,
295
296
  aws_session_token,