anyscale 0.26.69__py3-none-any.whl → 0.26.71__py3-none-any.whl

anyscale/service/_private/service_sdk.py

@@ -1,18 +1,28 @@
 import asyncio
 import copy
-from typing import Any, Dict, List, Optional, Union
+import json
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from anyscale._private.models.model_base import ResultIterator
 from anyscale._private.workload import WorkloadSDK
+from anyscale.client.openapi_client.models.apply_production_service_multi_version_v2_model import (
+    ApplyProductionServiceMultiVersionV2Model,
+)
 from anyscale.client.openapi_client.models.decorated_production_service_v2_api_model import (
     DecoratedProductionServiceV2APIModel,
 )
+from anyscale.client.openapi_client.models.decorated_production_service_v2_version_api_model import (
+    DecoratedProductionServiceV2VersionAPIModel,
+)
 from anyscale.client.openapi_client.models.decoratedlistserviceapimodel_list_response import (
     DecoratedlistserviceapimodelListResponse,
 )
 from anyscale.client.openapi_client.models.list_response_metadata import (
     ListResponseMetadata,
 )
+from anyscale.client.openapi_client.models.resource_tag_resource_type import (
+    ResourceTagResourceType,
+)
 from anyscale.compute_config.models import ComputeConfig
 from anyscale.sdk.anyscale_client.models import (
     AccessConfig,
@@ -21,6 +31,7 @@ from anyscale.sdk.anyscale_client.models import (
     ProductionServiceV2VersionModel,
     Protocols,
     RayGCSExternalStorageConfig as APIRayGCSExternalStorageConfig,
+    RolloutStrategy,
     ServiceConfig as ExternalAPIServiceConfig,
     ServiceEventCurrentState,
     ServiceSortField,
@@ -124,16 +135,21 @@ class PrivateServiceSDK(WorkloadSDK):
         canary_percent: Optional[int],
     ):
         """Log user-facing information about a deployed service."""
-        version_id_info = "version ID: {version_id}".format(
-            version_id=self._get_user_facing_service_version_id(
-                service.canary_version
-                if service.canary_version is not None
-                else service.primary_version
+        if not service.primary_version:
+            version_info = "versions: {versions}".format(
+                versions=[v.version for v in service.versions]
+            )
+        else:
+            version_info = "version ID: {version_id}".format(
+                version_id=self._get_user_facing_service_version_id(
+                    service.canary_version
+                    if service.canary_version is not None
+                    else service.primary_version
+                )
             )
-        )
         details = (
             "("
-            + version_id_info
+            + version_info
             + (
                 ")"
                 if canary_percent is None
@@ -270,6 +286,7 @@ class PrivateServiceSDK(WorkloadSDK):
         )
         return ApplyProductionServiceV2Model(
             name=name,
+            version=config.version_name,
             project_id=project_id,
             ray_serve_config=self._build_ray_serve_config(config),
             build_id=existing_config.build_id,
@@ -282,16 +299,19 @@ class PrivateServiceSDK(WorkloadSDK):
             ),
             ray_gcs_external_storage_config=existing_config.ray_gcs_external_storage_config,
             tracing_config=existing_config.tracing_config,
+            tags=getattr(config, "tags", None),
         )
 
     def _build_apply_service_model_for_rollout(  # noqa: PLR0912
         self,
         name: str,
         config: ServiceConfig,
+        rollout_strategy: RolloutStrategy,
         *,
         canary_percent: Optional[int] = None,
         max_surge_percent: Optional[int] = None,
         existing_service: Optional[DecoratedProductionServiceV2APIModel] = None,
+        traffic_percent: Optional[int] = None,
     ) -> ApplyProductionServiceV2Model:
         """Build the ApplyProductionServiceV2Model for a rolling update."""
 
@@ -399,13 +419,14 @@ class PrivateServiceSDK(WorkloadSDK):
 
         return ApplyProductionServiceV2Model(
             name=name,
+            version=config.version_name,
             project_id=project_id,
             ray_serve_config=self._build_ray_serve_config(config),
             build_id=build_id,
             compute_config_id=compute_config_id,
             canary_percent=canary_percent,
             max_surge_percent=max_surge_percent,
-            rollout_strategy="ROLLOUT",
+            rollout_strategy=rollout_strategy,
             config=ExternalAPIServiceConfig(
                 access=AccessConfig(use_bearer_token=config.query_auth_token_enabled),
                 protocols=Protocols(grpc=self._build_grpc_protocol_config(config)),
@@ -413,36 +434,327 @@ class PrivateServiceSDK(WorkloadSDK):
             ),
             ray_gcs_external_storage_config=ray_gcs_external_storage_config,
             tracing_config=tracing_config,
+            traffic_percent=traffic_percent,
+            tags=getattr(config, "tags", None),
         )
 
-    def deploy(  # noqa: PLR0912
+    def _build_apply_service_model_for_multi_version(
         self,
-        config: ServiceConfig,
+        name: str,
+        cloud: Optional[str],
+        project: Optional[str],
+        configs: List[ServiceConfig],
+        existing_service: Optional[DecoratedProductionServiceV2APIModel],
+        versions: List[Dict[str, Any]],
+    ) -> ApplyProductionServiceV2Model:
+        cloud_id = self.client.get_cloud_id(cloud_name=cloud)
+        project_id = self.client.get_project_id(parent_cloud_id=cloud_id, name=project)
+
+        # implement a map between version name and config file
+        # This way, we can iterate over the version config and corresponding config file to the _build_apply_service_model_for_rollout function
+        # Also, maybe add an assertion to confirm the version name is the same as the config file name.
+        # I also need to add a validation logic to confirm if all the version config passed(the version name) all has the corresponding config file.
+        version_service_map = self._get_service_version_map(configs, versions, existing_service)  # type: ignore
+        existing_versions: Dict[str, DecoratedProductionServiceV2VersionAPIModel] = {}
+        if existing_service is not None:
+            existing_versions = {
+                v.version: v
+                for v in self.client.get_service_versions(
+                    existing_service.id, read_all_versions=True
+                )
+            }
+
+        service_versions: List[ApplyProductionServiceV2Model] = []
+        for _, (config, version) in version_service_map.items():
+            version_name = version["name"]
+            traffic_percent = version["traffic_percent"]
+            if traffic_percent == 0:
+                self.logger.warning(
+                    f"Setting traffic for version {version_name} to 0 will terminate it."
+                )
+
+            # If config is None, it means the version is already deployed, and the user is likely to be only updating the traffic/capacity percent without serve config update.
+            if config is None:
+                if version_name not in existing_versions:
+                    raise ValueError(
+                        f"Version {version_name} does not exist in the existing service."
+                    )
+                existing_version = existing_versions[version_name]
+
+                service_version = ApplyProductionServiceV2Model(
+                    name=name,
+                    project_id=project_id,
+                    ray_serve_config=existing_version.ray_serve_config,
+                    build_id=existing_version.build_id,
+                    compute_config_id=existing_version.compute_config_id,
+                    rollout_strategy=RolloutStrategy.MULTI_VERSION,
+                    # TODO(doyoung): "config" is a service level config, so I'm assuming this is not necessary when this model is
+                    # representing a service version. If not, may need to add and store 'config' field in the version model, ProductionServiceV2VersionModel,
+                    # from the backend when the service version first gets created so it can be read from here for reconstruction.
+                    config=None,
+                    ray_gcs_external_storage_config=existing_version.ray_gcs_external_storage_config,
+                    tracing_config=existing_version.tracing_config,
+                    version=version_name,
+                    traffic_percent=traffic_percent,
+                )
+            else:
+                service_version = self._build_apply_service_model_for_rollout(
+                    name,
+                    config,
+                    RolloutStrategy.MULTI_VERSION,
+                    existing_service=existing_service,
+                    traffic_percent=traffic_percent,
+                )
+
+            service_versions.append(service_version)
+
+        assert len(service_versions) > 0, "service_versions should not be empty"
+        return ApplyProductionServiceMultiVersionV2Model(
+            rollout_strategy=RolloutStrategy.MULTI_VERSION,
+            cloud_id=cloud_id,
+            project_id=project_id,
+            service_versions=service_versions,
+        )
+
+    def _validate_versions(
+        self, versions: str
+    ) -> List[Dict[str, Any]]:  # noqa: C901, PLR0912
+        # Parse versions JSON string to list of dictionaries
+        try:
+            versions = json.loads(versions)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON format for --versions: {e}")
+
+        for version in versions:
+            # traffic_percent must be specified in all versions by the users.
+            # If some version does not specify traffic_percent, raise an error.
+            if "name" not in version:
+                raise ValueError("Name is required for each version.")
+            if "traffic_percent" not in version:
+                raise ValueError("Traffic percent is required for each version.")
+
+            # Verify the key names are valid. Only either of name, traffic_percent, or capacity_percent are allowed.
+            for key in version:
+                if key not in [
+                    "name",
+                    "traffic_percent",
+                ]:
+                    raise ValueError(f"Invalid key in versions: {key}")
+
+        if isinstance(versions, dict):
+            # Convert object form {"v1": 25, "v2": 75} to full list form
+            versions = [  # type: ignore
+                {"name": name, "traffic_percent": percent}
+                for name, percent in versions.items()
+            ]
+        elif not isinstance(versions, list):
+            raise ValueError(
+                "--versions must be a JSON array of objects or a JSON object in text format."
+            )
+
+        # Verify sum of each traffic_percent from each version is 100.
+        traffic_percent_sum = sum(version["traffic_percent"] for version in versions)
+        if traffic_percent_sum != 100:
+            raise ValueError("Sum of traffic percent for all versions must be 100.")
+
+        return versions  # type: ignore
+
+    def _read_and_validate_configs(
+        self, configs: List[ServiceConfig]
+    ) -> Tuple[str, Optional[str], Optional[str]]:
+        """The service name, cloud, and project must be identical for all versions."""
+
+        # If all the name field from configs are identical, use the first one as the name.
+        # If all the name field from configs are not provided, set to None, use the default name.
+        if all(config.name is None for config in configs):
+            name = self._get_default_name()
+        elif all(config.name == configs[0].name for config in configs):
+            name = configs[0].name
+        else:
+            raise ValueError("All provided names in config files must be identical.")
+
+        # If all the cloud field from configs are identical, use the first one as the cloud.
+        # If all the cloud field from configs are all set to None, set cloud to None. Default cloud will be used from backend.
+        if all(config.cloud is None for config in configs):
+            cloud = None
+        elif all(config.cloud == configs[0].cloud for config in configs):
+            cloud = configs[0].cloud
+        else:
+            raise ValueError("All provided clouds in config files must be identical.")
+
+        # If all the project field from configs are identical, use the first one as the project.
+        # If all the project field from configs are all set to None, set project to None. Default project will be used from backend.
+        if all(config.project is None for config in configs):
+            project = None
+        elif all(config.project == configs[0].project for config in configs):
+            project = configs[0].project
+        else:
+            raise ValueError("All provided projects in config files must be identical.")
+
+        return name, cloud, project
+
+    def _get_service_version_map(
+        self,
+        configs: List[ServiceConfig],
+        versions: Optional[List[Dict[str, Any]]],
+        existing_service: Optional[DecoratedProductionServiceV2APIModel],
+    ) -> Dict[str, Tuple[ServiceConfig, Dict[str, Any]]]:
+        service_config_to_version_config = {}
+        if versions is not None:
+            if len(configs) > len(versions):
+                raise ValueError(
+                    "Number of config files passed must be less than or equal to the number of version configs when multiple versions are being deployed."
+                )
+
+            for service_config in configs:
+                for version_config in versions:
+                    if service_config.version_name == version_config["name"]:
+                        service_config_to_version_config[version_config["name"]] = (
+                            service_config,
+                            version_config,
+                        )
+                        break
+
+                # version name in the ServiceConfig must have a matching version name in the version config.
+                # If this point is reached, then, it means there's a config file passed with version name that doesn't exist in the version config.
+                if service_config.version_name not in service_config_to_version_config:
+                    raise ValueError(
+                        f"version_name {service_config.version_name!r} in service config {service_config} does not match any of the version names provided with --versions."
+                    )
+
+            # version name in version config doesn't need to have a matching version name in service config. If there's not a matching, this
+            # means the version is already deployed, and the user is likely to be only updating the traffic/capacity percent.
+            for version_config in versions:
+                version_name = version_config["name"]
+                if version_name not in service_config_to_version_config:
+                    if existing_service is None:
+                        raise ValueError(
+                            f"--config-file corresponding to version {version_name!r} must be provided if the Service is not already deployed."
+                        )
+                    # TODO(doyoung): Perhaps add a validaiton logic to confirm if the version name already exists in the existing service through existing_service.versions.
+                    service_config_to_version_config[version_name] = (
+                        None,
+                        version_config,
+                    )
+
+        return service_config_to_version_config
+
+    def _get_rollout_strategy(  # noqa: PLR0912
+        self,
+        existing_service: Optional[DecoratedProductionServiceV2APIModel],
+        version_weights: Optional[List[Dict[str, Any]]],
+        in_place: bool,
+        configs: List[ServiceConfig],
+    ) -> RolloutStrategy:
+        """Determine the rollout strategy to use.
+
+        If `in_place` is True, return RolloutStrategy.IN_PLACE.
+        If `version_weights` is None or if the service is going from 0 or 1 versions to 1 version, return RolloutStrategy.ROLLOUT.
+        Otherwise, return RolloutStrategy.MULTI_VERSION.
+        """
+
+        active_versions = len(existing_service.versions) if existing_service else 0
+
+        if in_place:
+            if version_weights is not None:
+                raise ValueError(
+                    "In-place updates are not supported for multi-version services."
+                )
+
+            return RolloutStrategy.IN_PLACE
+        elif version_weights is None or (
+            len(version_weights) == 1 and active_versions <= 1
+        ):
+            if existing_service is None and len(configs) == 0:
+                raise ValueError(
+                    "A config file must be provided when deploying a new service."
+                )
+            if version_weights is not None:
+                self.logger.warning(
+                    "Defaulting to ROLLOUT strategy for single version service. Ignoring --versions."
+                )
+
+            return RolloutStrategy.ROLLOUT
+        else:
+            return RolloutStrategy.MULTI_VERSION
+
+    def deploy(  # noqa: PLR0912, C901
+        self,
+        configs: Union[ServiceConfig, List[ServiceConfig]],
         *,
         in_place: bool = False,
         canary_percent: Optional[int] = None,
         max_surge_percent: Optional[int] = None,
+        versions: Optional[str] = None,
+        name: Optional[str] = None,
+        cloud: Optional[str] = None,
+        project: Optional[str] = None,
     ) -> str:
-        if not isinstance(in_place, bool):
-            raise TypeError("in_place must be a bool.")
+        """Deploy or update a service.
 
-        if canary_percent is not None:
-            if not isinstance(canary_percent, int):
-                raise TypeError("canary_percent must be an int.")
-            if canary_percent < 0 or canary_percent > 100:
-                raise ValueError("canary_percent must be between 0 and 100.")
+        Args:
+            configs: Service configuration(s) to deploy. Can be a single ServiceConfig or a list of ServiceConfig objects for multi-version deployments.
+            versions: JSON string specifying version configurations for multi-version deployments.
+            name: Name of the service. If not provided, defaults to workspace cluster name.
+            cloud: Cloud provider to deploy to.
+            project: Project to deploy the service in.
 
-        if max_surge_percent is not None:
-            if not isinstance(max_surge_percent, int):
-                raise TypeError("max_surge_percent must be an int.")
-            if max_surge_percent < 0 or max_surge_percent > 100:
-                raise ValueError("max_surge_percent must be between 0 and 100.")
+        Returns:
+            The service ID.
+        """
+        if versions is not None:
+            versions = self._validate_versions(versions)  # type: ignore
 
-        name = config.name or self._get_default_name()
-        existing_service: Optional[DecoratedProductionServiceV2APIModel] = (
-            self.client.get_service(
-                name=name, cloud=config.cloud, project=config.project
+        if isinstance(configs, ServiceConfig):
+            if versions is not None:
+                raise ValueError(
+                    "To deploy with --versions, a list of ServiceConfig must be provided."
+                )
+
+            if not isinstance(in_place, bool):
+                raise TypeError("in_place must be a bool.")
+
+            if canary_percent is not None:
+                if not isinstance(canary_percent, int):
+                    raise TypeError("canary_percent must be an int.")
+                if canary_percent < 0 or canary_percent > 100:
+                    raise ValueError("canary_percent must be between 0 and 100.")
+
+            if max_surge_percent is not None:
+                if not isinstance(max_surge_percent, int):
+                    raise TypeError("max_surge_percent must be an int.")
+                if max_surge_percent < 0 or max_surge_percent > 100:
+                    raise ValueError("max_surge_percent must be between 0 and 100.")
+
+            name = configs.name
+            cloud = configs.cloud
+            project = configs.project
+            configs = [configs]
+
+        elif isinstance(configs, List):
+            if versions is None:
+                raise ValueError(
+                    "--versions must be provided when deploying with multiple config files."
+                )
+
+            if len(configs) == 0:
+                if name is None:
+                    raise ValueError(
+                        "When using --versions, a service name must be provided if no config files are given."
+                    )
+            else:
+                name, cloud, project = self._read_and_validate_configs(configs)
+
+        else:
+            ValueError(
+                "Single object of ServiceConfig or list of ServiceConfig must be provided."
             )
+
+        if name is None:
+            name = self._get_default_name()
+
+        existing_service: Optional[DecoratedProductionServiceV2APIModel] = (
+            self.client.get_service(name=name, cloud=cloud, project=project)
         )
         if existing_service is None:
             self.logger.info(f"Starting new service '{name}'.")
@@ -451,37 +763,63 @@ class PrivateServiceSDK(WorkloadSDK):
         else:
             self.logger.info(f"Updating existing service '{name}'.")
 
-        # passed canary_percent is ignored when creating or restarting a service
+        # passed canary_percent is ignored when creating or restarting a service or multiple versions are being deployed.
         is_new_or_restarting = (
             existing_service is None
             or existing_service.current_state == ServiceEventCurrentState.TERMINATED
         )
-        if canary_percent is not None and is_new_or_restarting:
+        if canary_percent is not None and (
+            is_new_or_restarting or versions is not None
+        ):
             canary_percent = None
             self.logger.warning(
-                "canary_percent is ignored when creating or restarting a service."
+                "canary_percent is ignored when creating or restarting a service or multiple versions are being deployed."
             )
 
-        if in_place:
+        rollout_strategy = self._get_rollout_strategy(
+            existing_service, versions, in_place, configs  # type: ignore
+        )
+
+        service: DecoratedProductionServiceV2APIModel
+        # TODO(doyoung): Support multi version deployment for in_place update.
+        if rollout_strategy == RolloutStrategy.IN_PLACE:
+            if versions is not None:
+                raise ValueError(
+                    "versions cannot be specified when doing an in_place update."
+                )
+            assert len(configs) == 1
             model = self._build_apply_service_model_for_in_place_update(
                 name,
-                config,
+                configs[0],
                 canary_percent=canary_percent,
                 max_surge_percent=max_surge_percent,
                 existing_service=existing_service,
             )
+            service = self.client.rollout_service(model)
+        elif rollout_strategy == RolloutStrategy.MULTI_VERSION:
+            model = self._build_apply_service_model_for_multi_version(
+                name,
+                cloud,
+                project,
+                configs,
+                existing_service=existing_service,
+                versions=versions,  # type: ignore
+            )
+            service = self.client.rollout_service_multi_version(model)
         else:
+            # rolling out a single version in new service without specifying versions.
+            assert len(configs) == 1
             model = self._build_apply_service_model_for_rollout(
                 name,
-                config,
+                configs[0],
+                RolloutStrategy.ROLLOUT,
                 canary_percent=canary_percent,
                 max_surge_percent=max_surge_percent,
                 existing_service=existing_service,
             )
+            service = self.client.rollout_service(model)
 
-        service = self.client.rollout_service(model)
         self._log_deployed_service_info(service, canary_percent=canary_percent)
-
         return service.id
 
     def _resolve_to_service_model(
@@ -619,14 +957,18 @@ class PrivateServiceSDK(WorkloadSDK):
                 sampling_ratio=model.tracing_config.sampling_ratio,
             )
 
+        version_name = model.version or self._get_user_facing_service_version_id(model)
+
         return ServiceVersionStatus(
             id=self._get_user_facing_service_version_id(model),
+            name=version_name,
             created_at=model.created_at,
             state=model.current_state,
             # NOTE(edoakes): there is also a "current_weight" field but it does not match the UI.
             weight=model.weight,
             config=ServiceConfig(
                 name=service_name,
+                version_name=version_name,
                 applications=model.ray_serve_config["applications"],
                 image_uri=str(image_uri),
                 compute_config=compute_config,
@@ -670,35 +1012,63 @@ class PrivateServiceSDK(WorkloadSDK):
         # which means that the per-version `query_auth_token_enabled` field will lie if
         # it's changed.
         query_auth_token_enabled = model.auth_token is not None
+        all_versions = None
+        primary_version = None
+        canary_version = None
+        project_name = None
 
-        primary_version_task = None
-        if model.primary_version is not None:
-            primary_version_task = asyncio.create_task(
-                self._service_version_model_to_status_async(
-                    model.primary_version,
-                    service_name=model.name,
-                    project_id=model.project_id,
-                    query_auth_token_enabled=query_auth_token_enabled,
+        if model.is_multi_version:
+            # For multi-version services, format all versions
+            version_tasks = [
+                asyncio.create_task(
+                    self._service_version_model_to_status_async(
+                        version,
+                        service_name=model.name,
+                        project_id=model.project_id,
+                        query_auth_token_enabled=query_auth_token_enabled,
+                    )
                 )
-            )
+                for version in model.versions
+            ]
+            all_versions = await asyncio.gather(*version_tasks)
+
+            if (
+                all_versions
+                and all_versions[0]
+                and isinstance(all_versions[0].config, ServiceConfig)
+            ):
+                project_name = all_versions[0].config.project
 
-        canary_version_task = None
-        if model.canary_version is not None:
-            canary_version_task = asyncio.create_task(
-                self._service_version_model_to_status_async(
-                    model.canary_version,
-                    service_name=model.name,
-                    project_id=model.project_id,
-                    query_auth_token_enabled=query_auth_token_enabled,
+        else:
+            primary_version_task = None
+            if model.primary_version is not None:
+                primary_version_task = asyncio.create_task(
+                    self._service_version_model_to_status_async(
+                        model.primary_version,
+                        service_name=model.name,
+                        project_id=model.project_id,
+                        query_auth_token_enabled=query_auth_token_enabled,
+                    )
                 )
-            )
 
-        primary_version = await primary_version_task if primary_version_task else None
-        canary_version = await canary_version_task if canary_version_task else None
+            canary_version_task = None
+            if model.canary_version is not None:
+                canary_version_task = asyncio.create_task(
+                    self._service_version_model_to_status_async(
+                        model.canary_version,
+                        service_name=model.name,
+                        project_id=model.project_id,
+                        query_auth_token_enabled=query_auth_token_enabled,
+                    )
+                )
 
-        project_name = None
-        if primary_version and isinstance(primary_version.config, ServiceConfig):
-            project_name = primary_version.config.project
+            primary_version = (
+                await primary_version_task if primary_version_task else None
+            )
+            canary_version = await canary_version_task if canary_version_task else None
+
+            if primary_version and isinstance(primary_version.config, ServiceConfig):
+                project_name = primary_version.config.project
 
         return ServiceStatus(
             id=model.id,
@@ -710,6 +1080,7 @@ class PrivateServiceSDK(WorkloadSDK):
             primary_version=primary_version,
             canary_version=canary_version,
             project=project_name,
+            versions=all_versions,
         )
 
     def status(
@@ -730,6 +1101,7 @@ class PrivateServiceSDK(WorkloadSDK):
         cloud: Optional[str] = None,
         project: Optional[str] = None,
         include_archived: bool = False,
+        tags_filter: Optional[Dict[str, List[str]]] = None,
         # Paging
         max_items: Optional[int] = None,  # Controls total items yielded by iterator
         page_size: Optional[int] = None,  # Controls items fetched per API call
@@ -770,6 +1142,18 @@ class PrivateServiceSDK(WorkloadSDK):
 
         normalised_states = _normalize_state_filter(state_filter)
 
+        # Convert tags dict into backend tag_filter list[str]
+        backend_tag_filter: Optional[List[str]] = None
+        if tags_filter:
+            flattened: List[str] = []
+            for key, values in tags_filter.items():
+                if not values:
+                    continue
+                for value in values:
+                    if key and value:
+                        flattened.append(f"{key}:{value}")
+            backend_tag_filter = flattened if len(flattened) > 0 else None
+
         def _fetch_page(
             token: Optional[str],
         ) -> DecoratedlistserviceapimodelListResponse:
@@ -780,6 +1164,7 @@ class PrivateServiceSDK(WorkloadSDK):
                 cloud=cloud,
                 project=project,
                 include_archived=include_archived,
+                tag_filter=backend_tag_filter,
                 count=page_size,
                 paging_token=token,
                 sort_field=sort_field,
@@ -873,6 +1258,44 @@ class PrivateServiceSDK(WorkloadSDK):
             model.primary_version, head, max_lines
         )
 
+    def add_tags(
+        self,
+        *,
+        id: Optional[str] = None,  # noqa: A002
+        name: Optional[str] = None,
+        cloud: Optional[str] = None,
+        project: Optional[str] = None,
+        tags: Dict[str, str],
+    ) -> None:
+        if id is None:
+            model = self._resolve_to_service_model(
+                name=name, cloud=cloud, project=project
+            )
+            resource_id = model.id
+        assert id is not None
+        self.client.upsert_resource_tags(
+            ResourceTagResourceType.SERVICE, resource_id, tags
+        )
+
+    def remove_tags(
+        self,
+        *,
+        id: Optional[str] = None,  # noqa: A002
+        name: Optional[str] = None,
+        cloud: Optional[str] = None,
+        project: Optional[str] = None,
+        keys: List[str],
+    ) -> None:
+        if id is None:
+            model = self._resolve_to_service_model(
+                name=name, cloud=cloud, project=project
+            )
+            resource_id = model.id
+        assert id is not None
+        self.client.delete_resource_tags(
+            ResourceTagResourceType.SERVICE, resource_id, keys
+        )
+
 
 def _normalize_state_filter(
     states: Optional[Union[List[ServiceState], List[str]]],