zenml-nightly 0.60.0.dev20240627__py3-none-any.whl → 0.61.0.dev20240711__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- README.md +30 -9
- RELEASE_NOTES.md +34 -0
- zenml/VERSION +1 -1
- zenml/analytics/enums.py +3 -0
- zenml/cli/__init__.py +28 -0
- zenml/cli/artifact.py +1 -2
- zenml/cli/integration.py +9 -8
- zenml/cli/server.py +6 -0
- zenml/cli/stack.py +797 -39
- zenml/cli/stack_components.py +7 -0
- zenml/cli/text_utils.py +35 -1
- zenml/cli/utils.py +127 -10
- zenml/client.py +23 -14
- zenml/config/docker_settings.py +8 -5
- zenml/constants.py +5 -1
- zenml/container_registries/base_container_registry.py +1 -0
- zenml/enums.py +6 -0
- zenml/event_hub/event_hub.py +5 -8
- zenml/integrations/aws/__init__.py +1 -0
- zenml/integrations/azure/__init__.py +1 -0
- zenml/integrations/deepchecks/__init__.py +1 -0
- zenml/integrations/discord/__init__.py +1 -0
- zenml/integrations/evidently/__init__.py +1 -0
- zenml/integrations/facets/__init__.py +1 -0
- zenml/integrations/feast/__init__.py +1 -0
- zenml/integrations/gcp/__init__.py +3 -1
- zenml/integrations/gcp/service_connectors/gcp_service_connector.py +203 -44
- zenml/integrations/huggingface/__init__.py +1 -0
- zenml/integrations/integration.py +24 -0
- zenml/integrations/kubeflow/__init__.py +3 -0
- zenml/integrations/kubeflow/flavors/kubeflow_orchestrator_flavor.py +1 -1
- zenml/integrations/kubeflow/orchestrators/kubeflow_orchestrator.py +0 -1
- zenml/integrations/kubernetes/__init__.py +3 -1
- zenml/integrations/kubernetes/orchestrators/kube_utils.py +4 -1
- zenml/integrations/label_studio/annotators/label_studio_annotator.py +1 -0
- zenml/integrations/langchain/__init__.py +1 -0
- zenml/integrations/mlflow/__init__.py +3 -1
- zenml/integrations/neural_prophet/__init__.py +1 -0
- zenml/integrations/polars/__init__.py +1 -0
- zenml/integrations/prodigy/__init__.py +1 -0
- zenml/integrations/pycaret/__init__.py +6 -0
- zenml/integrations/registry.py +37 -0
- zenml/integrations/s3/artifact_stores/s3_artifact_store.py +17 -6
- zenml/integrations/seldon/__init__.py +1 -0
- zenml/integrations/seldon/model_deployers/seldon_model_deployer.py +1 -0
- zenml/integrations/skypilot/flavors/skypilot_orchestrator_base_vm_config.py +2 -2
- zenml/integrations/skypilot/orchestrators/skypilot_base_vm_orchestrator.py +1 -1
- zenml/integrations/skypilot/orchestrators/skypilot_orchestrator_entrypoint.py +2 -2
- zenml/integrations/skypilot_aws/__init__.py +2 -1
- zenml/integrations/skypilot_azure/__init__.py +1 -1
- zenml/integrations/skypilot_gcp/__init__.py +1 -1
- zenml/integrations/skypilot_lambda/__init__.py +1 -1
- zenml/integrations/skypilot_lambda/flavors/skypilot_orchestrator_lambda_vm_flavor.py +1 -1
- zenml/integrations/slack/__init__.py +1 -0
- zenml/integrations/tekton/__init__.py +1 -0
- zenml/integrations/tensorboard/__init__.py +0 -1
- zenml/integrations/tensorflow/__init__.py +18 -6
- zenml/integrations/wandb/__init__.py +1 -0
- zenml/models/__init__.py +9 -0
- zenml/models/v2/core/component.py +18 -0
- zenml/models/v2/core/model.py +1 -2
- zenml/models/v2/core/service_connector.py +17 -0
- zenml/models/v2/core/stack.py +31 -0
- zenml/models/v2/misc/full_stack.py +97 -0
- zenml/models/v2/misc/stack_deployment.py +66 -0
- zenml/new/pipelines/pipeline.py +1 -1
- zenml/orchestrators/input_utils.py +3 -6
- zenml/stack/stack.py +3 -6
- zenml/stack_deployments/__init__.py +14 -0
- zenml/stack_deployments/aws_stack_deployment.py +289 -0
- zenml/stack_deployments/stack_deployment.py +130 -0
- zenml/stack_deployments/utils.py +40 -0
- zenml/utils/function_utils.py +1 -1
- zenml/utils/pagination_utils.py +7 -5
- zenml/utils/pipeline_docker_image_builder.py +97 -68
- zenml/utils/pydantic_utils.py +6 -5
- zenml/zen_server/cloud_utils.py +18 -3
- zenml/zen_server/dashboard/assets/{404-C1mcUujL.js → 404-DpJaNHKF.js} +1 -1
- zenml/zen_server/dashboard/assets/@radix-CFOkMR_E.js +85 -0
- zenml/zen_server/dashboard/assets/{@react-router-DYovave8.js → @react-router-CO-OsFwI.js} +2 -2
- zenml/zen_server/dashboard/assets/{@reactflow-DYIyhCfd.js → @reactflow-DJfzkHO1.js} +2 -2
- zenml/zen_server/dashboard/assets/@tanstack-DYiOyJUL.js +22 -0
- zenml/zen_server/dashboard/assets/AwarenessChannel-BYDLT2xC.js +1 -0
- zenml/zen_server/dashboard/assets/{CodeSnippet-WEzpO0az.js → CodeSnippet-BkOuRmyq.js} +2 -2
- zenml/zen_server/dashboard/assets/Commands-ZvWR1BRs.js +1 -0
- zenml/zen_server/dashboard/assets/CopyButton-DVwLkafa.js +2 -0
- zenml/zen_server/dashboard/assets/{CsvVizualization-Bx931j4U.js → CsvVizualization-C2IiqX4I.js} +7 -7
- zenml/zen_server/dashboard/assets/DisplayDate-DYgIjlDF.js +1 -0
- zenml/zen_server/dashboard/assets/EmptyState-BMLnFVlB.js +1 -0
- zenml/zen_server/dashboard/assets/Error-CqX0VqW_.js +1 -0
- zenml/zen_server/dashboard/assets/ExecutionStatus-BoLUXR9t.js +1 -0
- zenml/zen_server/dashboard/assets/Helpbox-LFydyVwh.js +1 -0
- zenml/zen_server/dashboard/assets/Infobox-DnENC0sh.js +1 -0
- zenml/zen_server/dashboard/assets/InlineAvatar-CbJtYr0t.js +1 -0
- zenml/zen_server/dashboard/assets/{MarkdownVisualization-DsB2QZiK.js → MarkdownVisualization-xp3hhULl.js} +2 -2
- zenml/zen_server/dashboard/assets/Pagination-DEbVUupy.js +1 -0
- zenml/zen_server/dashboard/assets/PasswordChecker-DUveqlva.js +1 -0
- zenml/zen_server/dashboard/assets/SetPassword-BYBdbQDo.js +1 -0
- zenml/zen_server/dashboard/assets/SuccessStep-Nx743hll.js +1 -0
- zenml/zen_server/dashboard/assets/{UpdatePasswordSchemas-CKrd3UZz.js → UpdatePasswordSchemas-DF9gSzE0.js} +1 -1
- zenml/zen_server/dashboard/assets/{aws-t0gKCj_R.js → aws-BgKTfTfx.js} +1 -1
- zenml/zen_server/dashboard/assets/{check-circle-BVvhm5dy.js → check-circle-i56092KI.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-down-zcvCWmyP.js → chevron-down-D_ZlKMqH.js} +1 -1
- zenml/zen_server/dashboard/assets/{chevron-right-double-CJ50E9Gr.js → chevron-right-double-BiEMg7rd.js} +1 -1
- zenml/zen_server/dashboard/assets/cloud-only-DVbIeckv.js +1 -0
- zenml/zen_server/dashboard/assets/{copy-BRhQz3j-.js → copy-BXNk6BjL.js} +1 -1
- zenml/zen_server/dashboard/assets/{database-CRRnyFWh.js → database-1xWSgZfO.js} +1 -1
- zenml/zen_server/dashboard/assets/{docker-BAonhm6G.js → docker-CQMVm_4d.js} +1 -1
- zenml/zen_server/dashboard/assets/{file-text-CbVERUON.js → file-text-CqD_iu6l.js} +1 -1
- zenml/zen_server/dashboard/assets/{help-B8rqCvqn.js → help-bu_DgLKI.js} +1 -1
- zenml/zen_server/dashboard/assets/index-C_CrU4vI.js +1 -0
- zenml/zen_server/dashboard/assets/index-DK1ynKjA.js +55 -0
- zenml/zen_server/dashboard/assets/index-inApY3KQ.css +1 -0
- zenml/zen_server/dashboard/assets/index-rK_Wuy2W.js +1 -0
- zenml/zen_server/dashboard/assets/index.esm-Corw4lXQ.js +1 -0
- zenml/zen_server/dashboard/assets/{login-mutation-Bk2tn324.js → login-mutation-BUnVASxp.js} +1 -1
- zenml/zen_server/dashboard/assets/not-found-B4VnX8gK.js +1 -0
- zenml/zen_server/dashboard/assets/package-CsUhPmou.js +1 -0
- zenml/zen_server/dashboard/assets/{page-D12Rvf0j.js → page-3efNCDeb.js} +2 -2
- zenml/zen_server/dashboard/assets/page-7zTHbhhI.js +1 -0
- zenml/zen_server/dashboard/assets/page-BEs6jK71.js +1 -0
- zenml/zen_server/dashboard/assets/page-BpSqIf4B.js +1 -0
- zenml/zen_server/dashboard/assets/{page-8vRWJ5b8.js → page-Bx6o0ARS.js} +1 -1
- zenml/zen_server/dashboard/assets/page-C43QGHTt.js +9 -0
- zenml/zen_server/dashboard/assets/page-CR0OG7ss.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CBuSUrE9.js → page-CRTJ0UuR.js} +1 -1
- zenml/zen_server/dashboard/assets/page-CUZIGO-3.js +1 -0
- zenml/zen_server/dashboard/assets/page-CaopxiU1.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CCtCgG-x.js → page-Cx67M0QT.js} +1 -1
- zenml/zen_server/dashboard/assets/page-D7Z399xy.js +1 -0
- zenml/zen_server/dashboard/assets/page-D93kd7Xj.js +1 -0
- zenml/zen_server/dashboard/assets/{page-Dw9-aJV6.js → page-DKlIdAe5.js} +1 -1
- zenml/zen_server/dashboard/assets/{page-COafKNbw.js → page-DMOYZppS.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DMsSn3dv.js +2 -0
- zenml/zen_server/dashboard/assets/{page-C6v3o0Qj.js → page-Dc_7KMQE.js} +1 -1
- zenml/zen_server/dashboard/assets/page-DvCvroOM.js +1 -0
- zenml/zen_server/dashboard/assets/page-Hus2pr9T.js +1 -0
- zenml/zen_server/dashboard/assets/page-JyfeDUfu.js +1 -0
- zenml/zen_server/dashboard/assets/{page-CH26py0a.js → page-Sxn82W-5.js} +1 -1
- zenml/zen_server/dashboard/assets/page-TKXERe16.js +1 -0
- zenml/zen_server/dashboard/assets/page-Xu8JEjSU.js +1 -0
- zenml/zen_server/dashboard/assets/{play-circle-DK5QMJyp.js → play-circle-CNtZKDnW.js} +1 -1
- zenml/zen_server/dashboard/assets/plus-DOeLmm7C.js +1 -0
- zenml/zen_server/dashboard/assets/{terminal-B2ovgWuz.js → terminal-By9cErXc.js} +1 -1
- zenml/zen_server/dashboard/assets/{update-server-settings-mutation-bKxf7U9h.js → update-server-settings-mutation-CR8e3Sir.js} +1 -1
- zenml/zen_server/dashboard/assets/{url-CgvM-IVM.js → url-DuQMeqYA.js} +1 -1
- zenml/zen_server/dashboard/assets/{zod-DrZvVLjd.js → zod-BhoGpZ63.js} +1 -1
- zenml/zen_server/dashboard/index.html +7 -7
- zenml/zen_server/dashboard_legacy/asset-manifest.json +4 -4
- zenml/zen_server/dashboard_legacy/index.html +1 -1
- zenml/zen_server/dashboard_legacy/{precache-manifest.e7c29295aae591541ef59d1734d79387.js → precache-manifest.c8c57fb0d2132b1d3c2119e776b7dfb3.js} +4 -4
- zenml/zen_server/dashboard_legacy/service-worker.js +1 -1
- zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js → main.382439a7.chunk.js} +2 -2
- zenml/zen_server/dashboard_legacy/static/js/{main.53857d8b.chunk.js.map → main.382439a7.chunk.js.map} +1 -1
- zenml/zen_server/deploy/helm/Chart.yaml +1 -1
- zenml/zen_server/deploy/helm/README.md +2 -2
- zenml/zen_server/feature_gate/zenml_cloud_feature_gate.py +11 -5
- zenml/zen_server/pipeline_deployment/utils.py +57 -44
- zenml/zen_server/rbac/zenml_cloud_rbac.py +11 -5
- zenml/zen_server/routers/stack_deployment_endpoints.py +144 -0
- zenml/zen_server/routers/workspaces_endpoints.py +64 -0
- zenml/zen_server/zen_server_api.py +2 -0
- zenml/zen_stores/migrations/utils.py +1 -1
- zenml/zen_stores/migrations/versions/0.61.0_release.py +23 -0
- zenml/zen_stores/migrations/versions/0d707865f404_adding_labels_to_stacks.py +30 -0
- zenml/zen_stores/rest_zen_store.py +117 -0
- zenml/zen_stores/schemas/stack_schemas.py +10 -0
- zenml/zen_stores/schemas/step_run_schemas.py +27 -11
- zenml/zen_stores/sql_zen_store.py +283 -0
- zenml/zen_stores/zen_store_interface.py +79 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/METADATA +32 -10
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/RECORD +175 -161
- zenml/zen_server/dashboard/assets/@radix-C9DBgJhe.js +0 -77
- zenml/zen_server/dashboard/assets/@tanstack-CEbkxrhX.js +0 -30
- zenml/zen_server/dashboard/assets/AwarenessChannel-B2KR83Tr.js +0 -1
- zenml/zen_server/dashboard/assets/Cards-DSEdjsk8.js +0 -1
- zenml/zen_server/dashboard/assets/Commands-CTlhyic5.js +0 -1
- zenml/zen_server/dashboard/assets/CopyButton-CTrzKmUO.js +0 -2
- zenml/zen_server/dashboard/assets/DisplayDate-BdguISQF.js +0 -1
- zenml/zen_server/dashboard/assets/EmptyState-BkooiGtL.js +0 -1
- zenml/zen_server/dashboard/assets/Error-4sKxHad4.js +0 -1
- zenml/zen_server/dashboard/assets/Helpbox-DW21i5LD.js +0 -1
- zenml/zen_server/dashboard/assets/Infobox-C7bf70VS.js +0 -1
- zenml/zen_server/dashboard/assets/InlineAvatar-Dxrtafpg.js +0 -1
- zenml/zen_server/dashboard/assets/PageHeader-B0pUife2.js +0 -1
- zenml/zen_server/dashboard/assets/Pagination-B9WG_9cJ.js +0 -1
- zenml/zen_server/dashboard/assets/PasswordChecker-DSLBp7Vl.js +0 -1
- zenml/zen_server/dashboard/assets/SetPassword-CiNhT15a.js +0 -1
- zenml/zen_server/dashboard/assets/SuccessStep-CykrFndS.js +0 -1
- zenml/zen_server/dashboard/assets/cloud-only-Bkawp7CJ.js +0 -1
- zenml/zen_server/dashboard/assets/index-BawkpTlr.js +0 -55
- zenml/zen_server/dashboard/assets/index-CRmm7QhS.css +0 -1
- zenml/zen_server/dashboard/assets/index.esm-F7nqy9zY.js +0 -1
- zenml/zen_server/dashboard/assets/not-found-BAuhP4Jb.js +0 -1
- zenml/zen_server/dashboard/assets/page--5YvAHg3.js +0 -1
- zenml/zen_server/dashboard/assets/page-B0RAq4s_.js +0 -1
- zenml/zen_server/dashboard/assets/page-BePtEPHl.js +0 -1
- zenml/zen_server/dashboard/assets/page-C1pra1Bc.js +0 -9
- zenml/zen_server/dashboard/assets/page-CSs4C9jL.js +0 -1
- zenml/zen_server/dashboard/assets/page-Cf2XSej0.js +0 -1
- zenml/zen_server/dashboard/assets/page-ClPUAE_f.js +0 -1
- zenml/zen_server/dashboard/assets/page-D8pf2vis.js +0 -1
- zenml/zen_server/dashboard/assets/page-DHKMmIQH.js +0 -1
- zenml/zen_server/dashboard/assets/page-DMZ0VOda.js +0 -1
- zenml/zen_server/dashboard/assets/page-Dcg-yQv_.js +0 -1
- zenml/zen_server/dashboard/assets/page-DoAK5FSB.js +0 -1
- zenml/zen_server/dashboard/assets/page-iXiDqE0J.js +0 -2
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/LICENSE +0 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/WHEEL +0 -0
- {zenml_nightly-0.60.0.dev20240627.dist-info → zenml_nightly-0.61.0.dev20240711.dist-info}/entry_points.txt +0 -0
@@ -43,7 +43,8 @@ from google.auth._default import (
|
|
43
43
|
_get_external_account_credentials,
|
44
44
|
)
|
45
45
|
from google.auth.transport.requests import Request
|
46
|
-
from google.cloud import container_v1, storage
|
46
|
+
from google.cloud import artifactregistry_v1, container_v1, storage
|
47
|
+
from google.cloud.location import locations_pb2
|
47
48
|
from google.oauth2 import credentials as gcp_credentials
|
48
49
|
from google.oauth2 import service_account as gcp_service_account
|
49
50
|
from pydantic import Field, field_validator, model_validator
|
@@ -834,14 +835,53 @@ GKE clusters in the GCP project that it is configured to use.
|
|
834
835
|
emoji=":cyclone:",
|
835
836
|
),
|
836
837
|
ResourceTypeModel(
|
837
|
-
name="GCP
|
838
|
+
name="GCP GAR container registry",
|
838
839
|
resource_type=DOCKER_REGISTRY_RESOURCE_TYPE,
|
839
840
|
description="""
|
840
|
-
Allows Stack Components to access a
|
841
|
+
Allows Stack Components to access a Google Artifact Registry as a standard
|
841
842
|
Docker registry resource. When used by Stack Components, they are provided a
|
842
843
|
pre-authenticated Python Docker client instance.
|
843
844
|
|
844
|
-
The configured credentials must have at least the following [GCP permissions](https://cloud.google.com/iam/docs/
|
845
|
+
The configured credentials must have at least the following [GCP permissions](https://cloud.google.com/iam/docs/understanding-roles#artifact-registry-roles):
|
846
|
+
|
847
|
+
- `artifactregistry.repositories.createOnPush`
|
848
|
+
- `artifactregistry.repositories.downloadArtifacts`
|
849
|
+
- `artifactregistry.repositories.get`
|
850
|
+
- `artifactregistry.repositories.list`
|
851
|
+
- `artifactregistry.repositories.readViaVirtualRepository`
|
852
|
+
- `artifactregistry.repositories.uploadArtifacts`
|
853
|
+
- `artifactregistry.locations.list`
|
854
|
+
|
855
|
+
The Artifact Registry Create-on-Push Writer role includes all of the above
|
856
|
+
permissions.
|
857
|
+
|
858
|
+
This resource type also includes legacy GCR container registry support.
|
859
|
+
|
860
|
+
**Important Notice: Google Container Registry** [**is being replaced by Artifact Registry**](https://cloud.google.com/artifact-registry/docs/transition/transition-from-gcr).
|
861
|
+
Please start using Artifact Registry for your containers. As per Google's
|
862
|
+
documentation, *"after May 15, 2024, Artifact Registry will host images for the
|
863
|
+
gcr.io domain in Google Cloud projects without previous Container Registry
|
864
|
+
usage. After March 18, 2025, Container Registry will be shut down."*.
|
865
|
+
|
866
|
+
Support for legacy GCR registries is still included in the GCP service
|
867
|
+
connector. Users that already have GCP service connectors configured to access
|
868
|
+
GCR registries may continue to use them without taking any action. However, it
|
869
|
+
is recommended to transition to Google Artifact Registries as soon as possible
|
870
|
+
by following [the GCP guide on this subject](https://cloud.google.com/artifact-registry/docs/transition/transition-from-gcr)
|
871
|
+
and making the following updates to ZenML GCP Service Connectors that are used
|
872
|
+
to access GCR resources:
|
873
|
+
|
874
|
+
* add the IAM permissions documented here to the GCP Service Connector
|
875
|
+
credentials to enable them to access the Artifact Registries.
|
876
|
+
* users may keep the gcr.io GCR URLs already configured in the GCP Service
|
877
|
+
Connectors as well as those used in linked Container Registry stack components
|
878
|
+
given that these domains are redirected by Google to GAR as covered in the GCR
|
879
|
+
transition guide. Alternatively, users may update the GCP Service Connector
|
880
|
+
configuration and/or the Container Registry stack components to use the
|
881
|
+
replacement Artifact Registry URLs.
|
882
|
+
|
883
|
+
When used with GCR registries, the configured credentials must have at least the
|
884
|
+
following [GCP permissions](https://cloud.google.com/iam/docs/understanding-roles#cloud-storage-roles):
|
845
885
|
|
846
886
|
- `storage.buckets.get`
|
847
887
|
- `storage.multipartUploads.abort`
|
@@ -855,17 +895,21 @@ The configured credentials must have at least the following [GCP permissions](ht
|
|
855
895
|
The Storage Legacy Bucket Writer role includes all of the above permissions
|
856
896
|
while at the same time restricting access to only the GCR buckets.
|
857
897
|
|
858
|
-
|
859
|
-
|
860
|
-
|
898
|
+
If set, the resource name must identify a GAR or GCR registry using one of the
|
899
|
+
following formats:
|
900
|
+
|
901
|
+
- Google Artifact Registry repository URI: `[https://]<region>-docker.pkg.dev/<project-id>/<registry-id>[/<repository-name>]`
|
902
|
+
- Google Artifact Registry name: `projects/<project-id>/locations/<location>/repositories/<repository-id>`
|
903
|
+
- (legacy) GCR repository URI: `[https://][us.|eu.|asia.]gcr.io/<project-id>[/<repository-name>]`
|
861
904
|
|
862
|
-
|
905
|
+
The connector can only be used to access GAR and GCR registries in the GCP
|
906
|
+
project that it is configured to use.
|
863
907
|
""",
|
864
908
|
auth_methods=GCPAuthenticationMethods.values(),
|
865
|
-
#
|
866
|
-
#
|
867
|
-
#
|
868
|
-
supports_instances=
|
909
|
+
# The connector provides access to the entire GCR container registry
|
910
|
+
# for the configured GCP project as well as any number of artifact
|
911
|
+
# registry repositories.
|
912
|
+
supports_instances=True,
|
869
913
|
logo_url="https://public-flavor-logos.s3.eu-central-1.amazonaws.com/container_registry/docker.png",
|
870
914
|
emoji=":whale:",
|
871
915
|
),
|
@@ -1142,60 +1186,101 @@ class GCPServiceConnector(ServiceConnector):
|
|
1142
1186
|
|
1143
1187
|
return bucket_name
|
1144
1188
|
|
1145
|
-
def
|
1189
|
+
def _parse_gar_resource_id(
|
1146
1190
|
self,
|
1147
1191
|
resource_id: str,
|
1148
|
-
) -> str:
|
1149
|
-
"""Validate and convert
|
1192
|
+
) -> Tuple[str, Optional[str]]:
|
1193
|
+
"""Validate and convert a GAR resource ID to a Google Artifact Registry ID and name.
|
1150
1194
|
|
1151
1195
|
Args:
|
1152
1196
|
resource_id: The resource ID to convert.
|
1153
1197
|
|
1154
1198
|
Returns:
|
1155
|
-
The
|
1199
|
+
The Google Artifact Registry ID and name. The name is omitted if the
|
1200
|
+
resource ID is a GCR repository URI.
|
1156
1201
|
|
1157
1202
|
Raises:
|
1158
|
-
ValueError: If the provided resource ID is not a valid
|
1159
|
-
repository URI.
|
1203
|
+
ValueError: If the provided resource ID is not a valid GAR
|
1204
|
+
or GCR repository URI.
|
1160
1205
|
"""
|
1161
1206
|
# The resource ID could mean different things:
|
1162
1207
|
#
|
1163
|
-
# -
|
1208
|
+
# - a GAR repository URI
|
1209
|
+
# - a GAR repository name
|
1210
|
+
# - a GCR repository URI (backwards-compatibility)
|
1164
1211
|
#
|
1165
1212
|
# We need to extract the project ID and registry ID from
|
1166
1213
|
# the provided resource ID
|
1167
1214
|
config_project_id = self.config.gcp_project_id
|
1168
1215
|
project_id: Optional[str] = None
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1216
|
+
canonical_url: str
|
1217
|
+
registry_name: Optional[str] = None
|
1218
|
+
|
1219
|
+
# A Google Artifact Registry URI uses the <location>-docker-pkg.dev
|
1220
|
+
# domain format with the project ID as the first part of the URL path
|
1221
|
+
# and the registry name as the second part of the URL path
|
1222
|
+
if match := re.match(
|
1223
|
+
r"^(https://)?(([a-z0-9-]+)-docker\.pkg\.dev/([a-z0-9-]+)/([a-z0-9-.]+))(/.+)*$",
|
1174
1224
|
resource_id,
|
1175
1225
|
):
|
1176
|
-
# The resource ID is a
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1226
|
+
# The resource ID is a Google Artifact Registry URI
|
1227
|
+
project_id = match[4]
|
1228
|
+
location = match[3]
|
1229
|
+
repository = match[5]
|
1230
|
+
|
1231
|
+
# Return the GAR URL without the image name and without the protocol
|
1232
|
+
canonical_url = match[2]
|
1233
|
+
registry_name = f"projects/{project_id}/locations/{location}/repositories/{repository}"
|
1234
|
+
|
1235
|
+
# Alternatively, the Google Artifact Registry name uses the
|
1236
|
+
# projects/<project-id>/locations/<location>/repositories/<repository-id>
|
1237
|
+
# format
|
1238
|
+
elif match := re.match(
|
1239
|
+
r"^projects/([a-z0-9-]+)/locations/([a-z0-9-]+)/repositories/([a-z0-9-.]+)$",
|
1240
|
+
resource_id,
|
1241
|
+
):
|
1242
|
+
# The resource ID is a Google Artifact Registry name
|
1243
|
+
project_id = match[1]
|
1244
|
+
location = match[2]
|
1245
|
+
repository = match[3]
|
1246
|
+
|
1247
|
+
# Return the GAR URL
|
1248
|
+
canonical_url = (
|
1249
|
+
f"{location}-docker.pkg.dev/{project_id}/{repository}"
|
1250
|
+
)
|
1251
|
+
registry_name = resource_id
|
1252
|
+
|
1253
|
+
# A legacy GCR repository URI uses one of several hostnames (gcr.io,
|
1254
|
+
# us.gcr.io, eu.gcr.io, asia.gcr.io) and the project ID is the
|
1255
|
+
# first part of the URL path
|
1256
|
+
elif match := re.match(
|
1257
|
+
r"^(https://)?(((us|eu|asia)\.)?gcr\.io/[a-z0-9-]+)(/.+)*$",
|
1258
|
+
resource_id,
|
1259
|
+
):
|
1260
|
+
# The resource ID is a legacy GCR repository URI.
|
1261
|
+
# Return the GAR URL without the image name and without the protocol
|
1262
|
+
canonical_url = match[2]
|
1263
|
+
|
1181
1264
|
else:
|
1182
1265
|
raise ValueError(
|
1183
|
-
f"Invalid resource ID for a
|
1184
|
-
f"Supported formats are:\n"
|
1266
|
+
f"Invalid resource ID for a Google Artifact Registry: "
|
1267
|
+
f"{resource_id}. Supported formats are:\n"
|
1268
|
+
f"Google Artifact Registry URI: [https://]<region>-docker.pkg.dev/<project-id>/<registry-id>[/<repository-name>]\n"
|
1269
|
+
f"Google Artifact Registry name: projects/<project-id>/locations/<location>/repositories/<repository-id>\n"
|
1185
1270
|
f"GCR repository URI: [https://][us.|eu.|asia.]gcr.io/<project-id>[/<repository-name>]"
|
1186
1271
|
)
|
1187
1272
|
|
1188
1273
|
# If the connector is configured with a project and the resource ID
|
1189
|
-
# is
|
1274
|
+
# is a GAR repository URI that specifies a different project,
|
1190
1275
|
# we raise an error
|
1191
1276
|
if project_id and project_id != config_project_id:
|
1192
1277
|
raise ValueError(
|
1193
|
-
f"The GCP project for the {resource_id}
|
1194
|
-
f"'{project_id}' does not match the project
|
1195
|
-
f"the connector: '{config_project_id}'."
|
1278
|
+
f"The GCP project for the {resource_id} Google Artifact "
|
1279
|
+
f"Registry '{project_id}' does not match the project "
|
1280
|
+
f"configured in the connector: '{config_project_id}'."
|
1196
1281
|
)
|
1197
1282
|
|
1198
|
-
return
|
1283
|
+
return canonical_url, registry_name
|
1199
1284
|
|
1200
1285
|
def _parse_gke_resource_id(self, resource_id: str) -> str:
|
1201
1286
|
"""Validate and convert an GKE resource ID to a GKE cluster name.
|
@@ -1244,7 +1329,7 @@ class GCPServiceConnector(ServiceConnector):
|
|
1244
1329
|
cluster_name = self._parse_gke_resource_id(resource_id)
|
1245
1330
|
return cluster_name
|
1246
1331
|
elif resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
|
1247
|
-
registry_id = self.
|
1332
|
+
registry_id, _ = self._parse_gar_resource_id(
|
1248
1333
|
resource_id,
|
1249
1334
|
)
|
1250
1335
|
return registry_id
|
@@ -1269,8 +1354,6 @@ class GCPServiceConnector(ServiceConnector):
|
|
1269
1354
|
"""
|
1270
1355
|
if resource_type == GCP_RESOURCE_TYPE:
|
1271
1356
|
return self.config.gcp_project_id
|
1272
|
-
elif resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
|
1273
|
-
return f"gcr.io/{self.config.gcp_project_id}"
|
1274
1357
|
|
1275
1358
|
raise RuntimeError(
|
1276
1359
|
f"Default resource ID not supported for '{resource_type}' resource "
|
@@ -1771,11 +1854,87 @@ class GCPServiceConnector(ServiceConnector):
|
|
1771
1854
|
raise AuthorizationException(msg) from e
|
1772
1855
|
|
1773
1856
|
if resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
|
1774
|
-
|
1857
|
+
# Get a GAR client
|
1858
|
+
gar_client = artifactregistry_v1.ArtifactRegistryClient(
|
1859
|
+
credentials=credentials
|
1860
|
+
)
|
1775
1861
|
|
1776
|
-
|
1777
|
-
|
1778
|
-
|
1862
|
+
if resource_id:
|
1863
|
+
registry_id, registry_name = self._parse_gar_resource_id(
|
1864
|
+
resource_id
|
1865
|
+
)
|
1866
|
+
|
1867
|
+
if registry_name is None:
|
1868
|
+
# This is a legacy GCR repository URI. We can't verify
|
1869
|
+
# the repository access without attempting to connect to it
|
1870
|
+
# via Docker/OCI, so just return the resource ID.
|
1871
|
+
return [registry_id]
|
1872
|
+
|
1873
|
+
# Check if the specified GAR registry exists
|
1874
|
+
try:
|
1875
|
+
repository = gar_client.get_repository(
|
1876
|
+
name=registry_name,
|
1877
|
+
)
|
1878
|
+
if repository.format_.name != "DOCKER":
|
1879
|
+
raise AuthorizationException(
|
1880
|
+
f"Google Artifact Registry '{resource_id}' is not a "
|
1881
|
+
"Docker registry."
|
1882
|
+
)
|
1883
|
+
return [registry_id]
|
1884
|
+
except google.api_core.exceptions.GoogleAPIError as e:
|
1885
|
+
msg = f"Failed to fetch Google Artifact Registry '{registry_id}': {e}"
|
1886
|
+
logger.error(msg)
|
1887
|
+
raise AuthorizationException(msg) from e
|
1888
|
+
|
1889
|
+
# For backwards compatibility, we initialize the list of resource
|
1890
|
+
# IDs with all GCR supported registries for the configured GCP
|
1891
|
+
# project
|
1892
|
+
resource_ids: List[str] = [
|
1893
|
+
f"{location}gcr.io/{self.config.gcp_project_id}"
|
1894
|
+
for location in ["", "us.", "eu.", "asia."]
|
1895
|
+
]
|
1896
|
+
|
1897
|
+
# List all Google Artifact Registries
|
1898
|
+
try:
|
1899
|
+
# First, we need to fetch all the Artifact Registry supported
|
1900
|
+
# locations
|
1901
|
+
locations = gar_client.list_locations(
|
1902
|
+
request=locations_pb2.ListLocationsRequest(
|
1903
|
+
name=f"projects/{self.config.gcp_project_id}"
|
1904
|
+
)
|
1905
|
+
)
|
1906
|
+
location_names = [
|
1907
|
+
locations.locations[i].location_id
|
1908
|
+
for i in range(len(locations.locations))
|
1909
|
+
]
|
1910
|
+
|
1911
|
+
# Then, we need to fetch all the repositories in each location
|
1912
|
+
repository_names: List[str] = []
|
1913
|
+
for location in location_names:
|
1914
|
+
repositories = gar_client.list_repositories(
|
1915
|
+
parent=f"projects/{self.config.gcp_project_id}/locations/{location}"
|
1916
|
+
)
|
1917
|
+
repository_names.extend(
|
1918
|
+
[
|
1919
|
+
repository.name
|
1920
|
+
for repository in repositories
|
1921
|
+
if repository.format_.name == "DOCKER"
|
1922
|
+
]
|
1923
|
+
)
|
1924
|
+
|
1925
|
+
for repository_name in repository_names:
|
1926
|
+
# Convert the repository name to a canonical GAR URL
|
1927
|
+
resource_ids.append(
|
1928
|
+
self._parse_gar_resource_id(repository_name)[0]
|
1929
|
+
)
|
1930
|
+
|
1931
|
+
except google.api_core.exceptions.GoogleAPIError as e:
|
1932
|
+
msg = f"Failed to list Google Artifact Registries: {e}"
|
1933
|
+
logger.error(msg)
|
1934
|
+
# TODO: enable when GCR is no longer supported:
|
1935
|
+
# raise AuthorizationException(msg) from e
|
1936
|
+
|
1937
|
+
return resource_ids
|
1779
1938
|
|
1780
1939
|
if resource_type == KUBERNETES_CLUSTER_RESOURCE_TYPE:
|
1781
1940
|
gke_client = container_v1.ClusterManagerClient(
|
@@ -1904,7 +2063,7 @@ class GCPServiceConnector(ServiceConnector):
|
|
1904
2063
|
if resource_type == DOCKER_REGISTRY_RESOURCE_TYPE:
|
1905
2064
|
assert resource_id is not None
|
1906
2065
|
|
1907
|
-
registry_id = self.
|
2066
|
+
registry_id, _ = self._parse_gar_resource_id(resource_id)
|
1908
2067
|
|
1909
2068
|
# Create a client-side Docker connector instance with the temporary
|
1910
2069
|
# Docker credentials
|
@@ -60,6 +60,7 @@ class Integration(metaclass=IntegrationMeta):
|
|
60
60
|
|
61
61
|
REQUIREMENTS: List[str] = []
|
62
62
|
APT_PACKAGES: List[str] = []
|
63
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL: List[str] = []
|
63
64
|
|
64
65
|
@classmethod
|
65
66
|
def check_installation(cls) -> bool:
|
@@ -143,6 +144,29 @@ class Integration(metaclass=IntegrationMeta):
|
|
143
144
|
"""
|
144
145
|
return cls.REQUIREMENTS
|
145
146
|
|
147
|
+
@classmethod
|
148
|
+
def get_uninstall_requirements(
|
149
|
+
cls, target_os: Optional[str] = None
|
150
|
+
) -> List[str]:
|
151
|
+
"""Method to get the uninstall requirements for the integration.
|
152
|
+
|
153
|
+
Args:
|
154
|
+
target_os: The target operating system to get the requirements for.
|
155
|
+
|
156
|
+
Returns:
|
157
|
+
A list of requirements.
|
158
|
+
"""
|
159
|
+
ret = []
|
160
|
+
for each in cls.get_requirements(target_os=target_os):
|
161
|
+
is_ignored = False
|
162
|
+
for ignored in cls.REQUIREMENTS_IGNORED_ON_UNINSTALL:
|
163
|
+
if each.startswith(ignored):
|
164
|
+
is_ignored = True
|
165
|
+
break
|
166
|
+
if not is_ignored:
|
167
|
+
ret.append(each)
|
168
|
+
return ret
|
169
|
+
|
146
170
|
@classmethod
|
147
171
|
def activate(cls) -> None:
|
148
172
|
"""Abstract method to activate the integration."""
|
@@ -31,6 +31,9 @@ class KubeflowIntegration(Integration):
|
|
31
31
|
|
32
32
|
NAME = KUBEFLOW
|
33
33
|
REQUIREMENTS = ["kfp>=2.6.0", "kfp-kubernetes>=1.1.0"] # Only 1.x version that supports pyyaml 6
|
34
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = [
|
35
|
+
"kfp", # it is used by GCP as well
|
36
|
+
]
|
34
37
|
|
35
38
|
@classmethod
|
36
39
|
def flavors(cls) -> List[Type[Flavor]]:
|
@@ -149,7 +149,7 @@ class KubeflowOrchestratorConfig(
|
|
149
149
|
|
150
150
|
kubeflow_hostname: Optional[str] = None
|
151
151
|
kubeflow_namespace: str = "kubeflow"
|
152
|
-
kubernetes_context: Optional[str] # TODO: Potential setting
|
152
|
+
kubernetes_context: Optional[str] = None # TODO: Potential setting
|
153
153
|
|
154
154
|
@model_validator(mode="before")
|
155
155
|
@classmethod
|
@@ -31,7 +31,9 @@ class KubernetesIntegration(Integration):
|
|
31
31
|
|
32
32
|
NAME = KUBERNETES
|
33
33
|
REQUIREMENTS = ["kubernetes>=21.7,<26"]
|
34
|
-
|
34
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = [
|
35
|
+
"kfp", # it is used by many others
|
36
|
+
]
|
35
37
|
@classmethod
|
36
38
|
def flavors(cls) -> List[Type[Flavor]]:
|
37
39
|
"""Declare the stack component flavors for the Kubernetes integration.
|
@@ -225,8 +225,11 @@ def wait_pod(
|
|
225
225
|
response = core_api.read_namespaced_pod_log(
|
226
226
|
name=pod_name,
|
227
227
|
namespace=namespace,
|
228
|
+
_preload_content=False,
|
228
229
|
)
|
229
|
-
|
230
|
+
raw_data = response.data
|
231
|
+
decoded_log = raw_data.decode("utf-8", errors="replace")
|
232
|
+
logs = decoded_log.splitlines()
|
230
233
|
if len(logs) > logged_lines:
|
231
234
|
for line in logs[logged_lines:]:
|
232
235
|
logger.info(line)
|
@@ -45,9 +45,11 @@ class MlflowIntegration(Integration):
|
|
45
45
|
# This downgrades pydantic to v1 even though mlflow does not have
|
46
46
|
# any issues with v2. This is why we have to pin it here so a downgrade
|
47
47
|
# will not happen.
|
48
|
-
"pydantic>=2.7.0,<2.8.0"
|
48
|
+
"pydantic>=2.7.0,<2.8.0",
|
49
49
|
]
|
50
50
|
|
51
|
+
REQUIREMENTS_IGNORED_ON_UNINSTALL = ["python-rapidjson", "pydantic"]
|
52
|
+
|
51
53
|
@classmethod
|
52
54
|
def activate(cls) -> None:
|
53
55
|
"""Activate the MLflow integration."""
|
zenml/integrations/registry.py
CHANGED
@@ -124,6 +124,43 @@ class IntegrationRegistry(object):
|
|
124
124
|
)
|
125
125
|
]
|
126
126
|
|
127
|
+
def select_uninstall_requirements(
|
128
|
+
self,
|
129
|
+
integration_name: Optional[str] = None,
|
130
|
+
target_os: Optional[str] = None,
|
131
|
+
) -> List[str]:
|
132
|
+
"""Select the uninstall requirements for a given integration or all integrations.
|
133
|
+
|
134
|
+
Args:
|
135
|
+
integration_name: Name of the integration to check.
|
136
|
+
target_os: Target OS for the requirements.
|
137
|
+
|
138
|
+
Returns:
|
139
|
+
List of requirements for the integration uninstall.
|
140
|
+
|
141
|
+
Raises:
|
142
|
+
KeyError: If the integration is not found.
|
143
|
+
"""
|
144
|
+
if integration_name:
|
145
|
+
if integration_name in self.list_integration_names:
|
146
|
+
return self._integrations[
|
147
|
+
integration_name
|
148
|
+
].get_uninstall_requirements(target_os=target_os)
|
149
|
+
else:
|
150
|
+
raise KeyError(
|
151
|
+
f"Version {integration_name} does not exist. "
|
152
|
+
f"Currently the following integrations are implemented. "
|
153
|
+
f"{self.list_integration_names}"
|
154
|
+
)
|
155
|
+
else:
|
156
|
+
return [
|
157
|
+
requirement
|
158
|
+
for name in self.list_integration_names
|
159
|
+
for requirement in self._integrations[
|
160
|
+
name
|
161
|
+
].get_uninstall_requirements(target_os=target_os)
|
162
|
+
]
|
163
|
+
|
127
164
|
def is_installed(self, integration_name: Optional[str] = None) -> bool:
|
128
165
|
"""Checks if all requirements for an integration are installed.
|
129
166
|
|
@@ -122,7 +122,7 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
|
|
122
122
|
|
123
123
|
def get_credentials(
|
124
124
|
self,
|
125
|
-
) -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
125
|
+
) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
|
126
126
|
"""Gets authentication credentials.
|
127
127
|
|
128
128
|
If an authentication secret is configured, the secret values are
|
@@ -130,8 +130,8 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
|
|
130
130
|
attributes.
|
131
131
|
|
132
132
|
Returns:
|
133
|
-
Tuple (key, secret, token) of credentials used to
|
134
|
-
the S3 filesystem.
|
133
|
+
Tuple (key, secret, token, region) of credentials used to
|
134
|
+
authenticate with the S3 filesystem.
|
135
135
|
|
136
136
|
Raises:
|
137
137
|
RuntimeError: If the AWS connector behaves unexpectedly.
|
@@ -151,6 +151,7 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
|
|
151
151
|
credentials.access_key,
|
152
152
|
credentials.secret_key,
|
153
153
|
credentials.token,
|
154
|
+
client.meta.region_name,
|
154
155
|
)
|
155
156
|
|
156
157
|
secret = self.get_typed_authentication_secret(
|
@@ -161,9 +162,10 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
|
|
161
162
|
secret.aws_access_key_id,
|
162
163
|
secret.aws_secret_access_key,
|
163
164
|
secret.aws_session_token,
|
165
|
+
None,
|
164
166
|
)
|
165
167
|
else:
|
166
|
-
return self.config.key, self.config.secret, self.config.token
|
168
|
+
return self.config.key, self.config.secret, self.config.token, None
|
167
169
|
|
168
170
|
@property
|
169
171
|
def filesystem(self) -> ZenMLS3Filesystem:
|
@@ -176,13 +178,22 @@ class S3ArtifactStore(BaseArtifactStore, AuthenticationMixin):
|
|
176
178
|
if self._filesystem and not self.connector_has_expired():
|
177
179
|
return self._filesystem
|
178
180
|
|
179
|
-
key, secret, token = self.get_credentials()
|
181
|
+
key, secret, token, region = self.get_credentials()
|
182
|
+
|
183
|
+
# Use the region from the connector if available, otherwise some
|
184
|
+
# remote workloads (e.g. Sagemaker) might not work correctly because
|
185
|
+
# they look for the bucket in the wrong region
|
186
|
+
client_kwargs = {}
|
187
|
+
if region:
|
188
|
+
client_kwargs["region_name"] = region
|
189
|
+
if self.config.client_kwargs:
|
190
|
+
client_kwargs.update(self.config.client_kwargs)
|
180
191
|
|
181
192
|
self._filesystem = ZenMLS3Filesystem(
|
182
193
|
key=key,
|
183
194
|
secret=secret,
|
184
195
|
token=token,
|
185
|
-
client_kwargs=
|
196
|
+
client_kwargs=client_kwargs,
|
186
197
|
config_kwargs=self.config.config_kwargs,
|
187
198
|
s3_additional_kwargs=self.config.s3_additional_kwargs,
|
188
199
|
)
|