anyscale 0.24.86__py3-none-any.whl

anyscale/utils/cloud_utils.py

@@ -0,0 +1,317 @@
+import contextlib
+from enum import Enum
+import sys
+import time
+from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar
+import uuid
+
+import boto3
+from botocore.exceptions import ClientError
+from click.exceptions import ClickException
+
+from anyscale.cli_logger import CloudSetupLogger
+from anyscale.client.openapi_client.api.default_api import DefaultApi
+from anyscale.client.openapi_client.models import (
+    CloudAnalyticsEvent,
+    CloudAnalyticsEventCloudResource,
+    CloudAnalyticsEventCommandName,
+    CloudAnalyticsEventError,
+    CloudAnalyticsEventName,
+    CloudProviders,
+    CreateAnalyticsEvent,
+)
+from anyscale.cloud import get_cloud_id_and_name
+from anyscale.shared_anyscale_utils.utils.collections import flatten
+
+
+V = TypeVar("V")
+
+
+def get_organization_default_cloud(api_client: DefaultApi) -> Optional[str]:
+    """Return default cloud name for organization if it exists and
+        if user has correct permissions for it.
+
+        Returns:
+            Name of default cloud name for organization if it exists and
+            if user has correct permissions for it.
+        """
+    user = api_client.get_user_info_api_v2_userinfo_get().result
+    organization = user.organizations[0]  # Each user only has one org
+    if organization.default_cloud_id:
+        try:
+            # Check permissions
+            _, cloud_name = get_cloud_id_and_name(
+                api_client, cloud_id=organization.default_cloud_id
+            )
+            return str(cloud_name)
+        except Exception:  # noqa: BLE001
+            return None
+    return None
+
+
+def get_default_cloud(
+    api_client: DefaultApi, cloud_name: Optional[str]
+) -> Tuple[str, str]:
+    """Returns the cloud id from cloud name.
+    If cloud name is not provided, returns the default cloud name if exists in organization.
+    If default cloud name does not exist returns last used cloud.
+    """
+
+    if cloud_name is None:
+        default_cloud_name = get_organization_default_cloud(api_client)
+        if default_cloud_name:
+            cloud_name = default_cloud_name
+    return get_cloud_id_and_name(api_client, cloud_name=cloud_name)
+
+
+def verify_anyscale_access(
+    api_client: DefaultApi,
+    cloud_id: str,
+    cloud_provider: CloudProviders,
+    logger: CloudSetupLogger,
+) -> bool:
+    try:
+        api_client.verify_access_api_v2_cloudsverify_access_cloud_id_get(cloud_id)
+        return True
+    except ClickException as e:
+        if cloud_provider == CloudProviders.AWS:
+            logger.log_resource_error(
+                CloudAnalyticsEventCloudResource.AWS_IAM_ROLE,
+                CloudSetupError.ANYSCALE_ACCESS_DENIED,
+            )
+        elif cloud_provider == CloudProviders.GCP:
+            logger.log_resource_error(
+                CloudAnalyticsEventCloudResource.GCP_SERVICE_ACCOUNT,
+                CloudSetupError.ANYSCALE_ACCESS_DENIED,
+            )
+        logger.error(
+            f"Anyscale's control plane is unable to access resources on your cloud provider.\n{e}"
+        )
+        return False
+
+
+def modify_memorydb_parameter_group(
+    parameter_group_name: str,
+    region: str,
+    boto3_session: Optional[boto3.Session] = None,
+) -> None:
+    """
+    Modify the memorydb parameter group to set the maxmemory-policy to allkeys-lru.
+
+    This is not done in the cloudformation template because we have to create the paramter group first and then modify it.
+    """
+    try:
+        if boto3_session is None:
+            boto3_session = boto3.Session()
+        memorydb_client = boto3_session.client("memorydb", region_name=region)
+        memorydb_client.update_parameter_group(
+            ParameterGroupName=parameter_group_name,
+            ParameterNameValues=[
+                {"ParameterName": "maxmemory-policy", "ParameterValue": "allkeys-lru",}
+            ],
+        )
+    except ClientError as e:
+        # TODO (allenyin): add memorydb to the cloud provider error list.
+        raise ClickException(
+            f"Failed to modify memorydb parameter group {parameter_group_name}. Please make sure you have permission to perform UpdateParameterGroup on memorydb clusters and try again. \n{e}"
+        )
+
+
+def wait_for_gcp_lb_resource_termination(
+    api_client: DefaultApi,
+    cloud_id: str,
+    timeout_s: int = 900,  # 15 minute timeout
+    poll_interval_s: int = 10,  # Poll every 10 seconds
+):
+    start = time.time()
+    while time.time() - start < timeout_s:
+        response = api_client.get_lb_resource_api_v2_clouds_with_cloud_resource_gcp_router_cloud_id_get_lb_resource_post(
+            cloud_id=cloud_id
+        ).result
+        if response.is_terminated:
+            break
+        time.sleep(poll_interval_s)
+    else:
+        raise ClickException(
+            f"LB resources and namespace termination timed out after {timeout_s} seconds."
+        )
+
+
+class CloudSetupError(str, Enum):
+    ANYSCALE_ACCESS_DENIED = "ANYSCALE_ACCESS_DENIED"
+    RESOURCE_NOT_FOUND = "RESOURCE_NOT_FOUND"
+    CIDR_BLOCK_TOO_SMALL = "CIDR_BLOCK_TOO_SMALL"
+    MISSING_CLOUD_RESOURCE_ID = "MISSING_CLOUD_RESOURCE_ID"
+    ONLY_ONE_SUBNET = "ONLY_ONE_SUBNET"
+    SUBNET_NOT_IN_VPC = "SUBNET_NOT_IN_VPC"
+    ONLY_ONE_AZ = "ONLY_ONE_AZ"
+    IAM_ROLE_ACCOUNT_MISMATCH = "IAM_ROLE_ACCOUNT_MISMATCH"
+    INSTANCE_PROFILE_NOT_FOUND = "INSTANCE_PROFILE_NOT_FOUND"
+    INTERNAL_COMMUNICATION_NOT_ALLOWED = "INTERNAL_COMMUNICATION_NOT_ALLOWED"
+    MALFORMED_CORS_RULE = "MALFORMED_CORS_RULE"
+    INCORRECT_CORS_RULE = "INCORRECT_CORS_RULE"
+    MOUNT_TARGET_NOT_FOUND = "MOUNT_TARGET_NOT_FOUND"
+    INVALID_MOUNT_TARGET = "INVALID_MOUNT_TARGET"
+    PROJECT_NOT_ACTIVE = "PROJECT_NOT_ACTIVE"
+    API_NOT_ENABLED = "API_NOT_ENABLED"
+    FIREWALL_NOT_ASSOCIATED_WITH_VPC = "FIREWALL_NOT_ASSOCIATED_WITH_VPC"
+    FILESTORE_NAME_MALFORMED = "FILESTORE_NAME_MALFORMED"
+    FILESTORE_NOT_CONNECTED_TO_VPC = "FILESTORE_NOT_CONNECTED_TO_VPC"
+    MEMORYSTORE_NOT_CONNECTED_TO_VPC = "MEMORYSTORE_NOT_CONNECTED_TO_VPC"
+    MEMORYSTORE_READ_REPLICAS_DISABLED = "MEMORYSTORE_READ_REPLICAS_DISABLED"
+    MEMORYDB_CLUSTER_UNAVAILABLE = "MEMORYDB_CLUSTER_UNAVAILABLE"
+
+
+class CloudEventProducer:
+    """
+    Produce events during cloud setup/register/verify
+    """
+
+    def __init__(self, cli_version: str, api_client: DefaultApi):
+        self.api_client = api_client
+        self.cloud_id: Optional[str] = None
+        self.cloud_provider: Optional[CloudProviders] = None
+        self.cli_version = cli_version
+
+    def init_trace_context(
+        self,
+        command_name: CloudAnalyticsEventCommandName,
+        cloud_provider: CloudProviders,
+        cloud_id: Optional[str] = None,
+    ):
+        self.trace_id = str(uuid.uuid4().hex)
+        self.command_name = command_name
+        self.raw_command_input = str(" ".join(sys.argv[1:]))
+        self.cloud_id = cloud_id
+        self.cloud_provider = cloud_provider
+
+    def set_cloud_id(self, cloud_id: str):
+        self.cloud_id = cloud_id
+
+    def produce(
+        self,
+        event_name: CloudAnalyticsEventName,
+        succeeded: bool,
+        logger: Optional[CloudSetupLogger] = None,
+        internal_error: Optional[str] = None,
+    ):
+        with contextlib.suppress(Exception):
+            # shouldn't block cloud setup even if cloud event generation fails
+            error = None
+            if not succeeded:
+                cloud_provider_error = None
+                if logger:
+                    cloud_provider_error_list = logger.get_cloud_provider_errors()
+                    logger.clear_cloud_provider_errors()
+                    if len(cloud_provider_error_list) > 0:
+                        cloud_provider_error = cloud_provider_error_list
+                error = CloudAnalyticsEventError(
+                    internal_error=internal_error,
+                    cloud_provider_error=cloud_provider_error,
+                )
+
+            self.api_client.produce_analytics_event_api_v2_analytics_post(
+                CreateAnalyticsEvent(
+                    cloud_analytics_event=CloudAnalyticsEvent(
+                        cli_version=self.cli_version,
+                        trace_id=self.trace_id,
+                        cloud_id=self.cloud_id,
+                        succeeded=succeeded,
+                        command_name=self.command_name,
+                        raw_command_input=self.raw_command_input,
+                        cloud_provider=self.cloud_provider,
+                        event_name=event_name,
+                        error=error,
+                    )
+                )
+            )
+
+
+def validate_aws_credentials(
+    logger: CloudSetupLogger, boto3_session: Optional[Any] = None
+) -> bool:
+    try:
+        if boto3_session is None:
+            boto3_session = boto3.Session()
+        boto3.client("sts").get_caller_identity()
+        return True
+    except Exception as e:  # noqa: BLE001
+        logger.error(f"Failed to validate AWS credentials: {e}")
+        return False
+
+
+def get_errored_resources_and_reasons(
+    cfn_client: Any, stack_name: str
+) -> Dict[str, str]:
+    """
+    Describes the CloudFormation stack events and extracts the failure reasons.
+    """
+    error_details: Dict[str, str] = {}
+    response = cfn_client.describe_stack_events(StackName=stack_name)
+    error_details.update(
+        extract_cloudformation_failure_reasons(response["StackEvents"])
+    )
+    while response.get("NextToken") is not None:
+        response = cfn_client.describe_stack_events(
+            StackName=stack_name, NextToken=response["NextToken"]
+        )
+        error_details.update(
+            extract_cloudformation_failure_reasons(response["StackEvents"])
+        )
+    return error_details
+
+
+def extract_cloudformation_failure_reasons(
+    events: List[Dict[str, Any]]
+) -> Dict[str, str]:
+    """
+    Extracts the failure reasons from the CloudFormation events.
+    """
+    failure_reasons: Dict[str, str] = {}
+    for event in events:
+        if event.get("ResourceStatus") == "CREATE_FAILED":
+            error_resource = event.get("LogicalResourceId", "UnknownResource")
+            error_reason = event.get("ResourceStatusReason", "Unknown reason")
+            if "Resource creation cancelled" in error_reason:
+                # Resource creation cancelled due to other resource failures.
+                # Skip this error.
+                continue
+            failure_reasons[error_resource] = error_reason
+    return failure_reasons
+
+
+def _unroll_resources_for_aws_list_call(aws_list_function: Callable, list_key: str):
+    def token_extractor(response) -> Optional[str]:
+        return response.get("NextToken")
+
+    # NOTE: `NextToken` is expected to have `str` type and therefore we have
+    #       we have to workaround its optionality by completely omitting it
+    #       from the call
+    responses = unroll_pagination(
+        lambda next_token: aws_list_function(
+            **({"NextToken": next_token} if next_token else {}),  # type: ignore
+        ),
+        token_extractor,
+    )
+
+    return flatten(*[r[list_key] for r in responses if list_key in r])
+
+
+def unroll_pagination(
+    paginated_method: Callable, next_token_extractor: Callable,
+):
+    """Handles paginated method's invocation by
+    - Repeatedly invoking the method (injecting the token from its previous response)
+    - Collecting all of the responses in a list
+    """
+
+    next_token: Optional[str] = None
+    results = []
+    while True:
+        r = paginated_method(next_token)
+        results.append(r)
+        next_token = next_token_extractor(r)
+        if not next_token:
+            break
+
+    return results

anyscale/utils/cluster_debug.py

@@ -0,0 +1,191 @@
+from dataclasses import dataclass
+import socket
+import ssl
+from typing import Callable, Dict, Iterator, Optional
+
+import requests
+
+from anyscale.cli_logger import BlockLogger
+
+
+@dataclass
+class DebugTestResult:
+    name: str
+    success: bool
+    error: Optional[Exception] = None
+    warnings: Optional[str] = None
+
+
+def debug_cluster(
+    logger: BlockLogger, cluster_domain_name: str, anyscale_ip: str
+) -> Iterator[DebugTestResult]:
+    has_experienced_a_failure: bool = False
+
+    tests = list(_tests_for(cluster_domain_name, anyscale_ip).items())
+
+    for name, test in tests:
+        try:
+            warning = test()
+
+            yield DebugTestResult(name, True, warnings=warning)
+        except Exception as e:  # noqa: BLE001
+            yield DebugTestResult(name, False, e)
+
+            if not has_experienced_a_failure:
+                has_experienced_a_failure = True
+
+                try:
+                    cluster_ip_addr = socket.gethostbyname(cluster_domain_name)
+                except Exception as e:  # noqa: BLE001
+                    resp = requests.get(
+                        "https://dns.google/resolve",
+                        params={"name": cluster_domain_name},
+                    )
+                    resp.raise_for_status()
+
+                    resp_json = resp.json()
+                    cluster_ip_addr = resp_json["Answer"][0]["data"]
+                logger.warning(
+                    "Failure detected, retrying tests by directly communicating with the IP address."
+                )
+                tests.extend(
+                    _tests_for(
+                        cluster_ip_addr, anyscale_ip, tls_sni=cluster_domain_name
+                    ).items()
+                )
+
+
+def _tests_for(
+    endpoint: str, anyscale_ip: str, tls_sni: Optional[str] = None
+) -> Dict[str, Callable[[], Optional[str]]]:
+    """Attempt to debug networking for an Anyscale Cluster.
+
+    Arguments:
+        endpoint: Hostname or IP address of the cluster
+        tls_sni: Hostname to use for TLS SNI (used if used to directly dial the IP address)
+
+    Also ask for the IP from the control Plane
+
+    Fallbacks:
+    1. Try directly using different IPs + hostname
+
+    """
+    return {
+        **(
+            {
+                "dns-lookup": lambda: _dns_test(endpoint, anyscale_ip),
+                "dns-over-https": lambda: _dns_over_https_test(endpoint, anyscale_ip),
+            }
+            if tls_sni is None
+            else {}
+        ),
+        "tcp-connect": lambda: _tcp_test(endpoint),
+        "tls-no-sni": lambda: _tls_no_sni(endpoint),
+        "tls-connect": lambda: _tls_test(endpoint, tls_sni),
+        "https-get": lambda: _https_test(endpoint, tls_sni),
+    }
+
+
+def _dns_test(endpoint: str, anyscale_ip: str) -> Optional[str]:
+    resolved_ip = socket.gethostbyname(endpoint)
+    if resolved_ip != anyscale_ip:
+        return (
+            f"Unexpected local DNS resolution:\nLocal DNS resolves this to {resolved_ip}\n"
+            f"Anyscale expects this host to resolve to {anyscale_ip}\n"
+            "This is safe to ignore if you are using a VPN to connect to your cluster.\n"
+        )
+    return None
+
+
+def _dns_over_https_test(endpoint: str, anyscale_ip: str) -> Optional[str]:
+    resp = requests.get("https://dns.google/resolve", params={"name": endpoint})
+    resp.raise_for_status()
+
+    resp_json = resp.json()
+    answers = resp_json.get("Answer", [])
+    assert len(answers) > 0, "Got no answers for a successful query"
+
+    resolved_ips = {answer["data"] for answer in answers}
+    if anyscale_ip not in resolved_ips:
+        return (
+            f"Unexpected Public DNS resolution. This should never happen\n"
+            f"`8.8.8.8` resolves this host to {resolved_ips}\n"
+            f"Anyscale expects this host to resolve to {anyscale_ip}\n"
+        )
+    return None
+
+
+def _tcp_test(endpoint: str) -> None:
+    with socket.create_connection((endpoint, 443)):
+        pass
+
+
+def _tls_test(endpoint: str, hostname: Optional[str] = None) -> None:
+    """Create a TLS connection to the destination.
+
+        The optional hostname is used if the TLS connection is made directly to an IP address
+        and we want to set the SNI separately.
+    """
+    if hostname is None:
+        hostname = endpoint
+
+    with socket.create_connection(
+        (endpoint, 443)
+    ) as sock, ssl.create_default_context().wrap_socket(sock, server_hostname=hostname):
+        pass
+
+
+def _tls_no_sni(endpoint: str) -> None:
+    """Create a TLS connection to the destination without specifying an SNI.
+
+    This is used to test if middleboxes/firewalls are rejecting connections based on SNI.
+    """
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    with socket.create_connection((endpoint, 443)) as sock, ctx.wrap_socket(
+        sock, server_hostname=None
+    ):
+        pass
+
+
+def _https_test(endpoint: str, hostname: Optional[str] = None) -> None:
+    if hostname is None:
+        hostname = endpoint
+
+    session = requests.Session()
+    session.mount(f"https://{endpoint}", HostnameSpecificAdapter(hostname))
+    resp = session.get(f"https://{endpoint}")
+
+    if resp.status_code != requests.codes["unauthorized"]:
+        raise RuntimeError(
+            f"Unexpected status code from https://{endpoint}: {resp.status_code}"
+        )
+
+    assert (
+        "x-anyscale-authentication-status" in resp.headers
+        and resp.headers["x-anyscale-authentication-status"] == "Unauthorized"
+    ), "Failed to get an expected anyscale-unique header from the response."
+
+
+class HostnameSpecificAdapter(requests.adapters.HTTPAdapter):
+    """Adapter to handle directly connecting to an IP address with a specific TLS Hostname.
+
+    This allows requests to be made to https://<ip address>, and still have TLS work for the server name.
+    """
+
+    def __init__(self, hostname: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        class SingleHostSSLContext(ssl.SSLContext):
+            def wrap_socket(self, *args, **kwargs):
+                kwargs["server_hostname"] = hostname
+                return super().wrap_socket(*args, **kwargs)
+
+        self.init_poolmanager(
+            self._pool_connections,  # type:ignore
+            self._pool_maxsize,  # type:ignore
+            ssl_context=SingleHostSSLContext(),
+            assert_hostname=hostname,
+        )

anyscale/utils/connect_helpers.py

@@ -0,0 +1,155 @@
+from dataclasses import dataclass
+import inspect
+from typing import Any, Callable, Dict, Generic, List, Optional, TYPE_CHECKING, TypeVar
+
+from anyscale.anyscale_pydantic import BaseModel
+from anyscale.client.openapi_client.models.session import Session
+from anyscale.sdk.anyscale_client.models.cluster import Cluster
+
+
+if TYPE_CHECKING:
+    from anyscale.sdk.anyscale_client.sdk import AnyscaleSDK
+
+
+try:
+    from ray.client_builder import ClientContext
+except ImportError:
+    # Import error will be raised in `ClientBuilder.__init__`.
+    # Not raising error here to allow `anyscale.connect required_ray_version`
+    # to work.
+    @dataclass
+    class ClientContext:  # type: ignore
+        pass
+
+
+@dataclass
+class AnyscaleClientConnectResponse:
+    """
+    Additional information returned about clusters that were connected from Anyscale.
+    """
+
+    cluster_id: str
+
+
+class AnyscaleClientContext(ClientContext):  # type: ignore
+    def __init__(
+        self, anyscale_cluster_info: AnyscaleClientConnectResponse, **kwargs: Any,
+    ) -> None:
+        if _multiclient_supported() and "_context_to_restore" not in kwargs:
+            # Set to None for now until multiclient is supported on connect
+            kwargs["_context_to_restore"] = None
+        super().__init__(**kwargs)
+        self.anyscale_cluster_info = anyscale_cluster_info
+
+
+def _multiclient_supported() -> bool:
+    """True if ray version supports multiple clients, False otherwise"""
+    _context_params = inspect.signature(ClientContext.__init__).parameters
+    return "_context_to_restore" in _context_params
+
+
+def find_project_id(sdk: "AnyscaleSDK", project_name: str) -> Optional[str]:
+    """Return id if a project of a given name exists."""
+    resp = sdk.search_projects({"name": {"equals": project_name}})
+    if len(resp.results) > 0:
+        return resp.results[0].id  # type: ignore
+    else:
+        return None
+
+
+def get_cluster(
+    sdk: "AnyscaleSDK", project_id: str, session_name: str
+) -> Optional[Session]:
+    """Query Anyscale for the given cluster's metadata."""
+    clusters = sdk.search_clusters(
+        {
+            "project_id": project_id,
+            "name": {"equals": session_name},
+            "archive_status": "ALL",
+        }
+    ).results
+    cluster_found: Optional[Cluster] = None
+    for cluster in clusters:
+        if cluster.name == session_name:
+            cluster_found = cluster
+            break
+
+    # A separate call is needed because downstream clients expect "Session" type
+    if cluster_found:
+        return sdk.get_session(cluster_found.id).result
+    return None
+
+
+def list_entities(
+    list_function: Callable[..., Any],
+    container_id: Optional[str] = None,
+    filters: Optional[Dict[str, Any]] = None,
+    max: Optional[int] = None,  # noqa: A002
+) -> List[Any]:
+    """Convenience function to automatically handle paging tokens.
+
+    This function repeatedly calls `list_function`, which is a GET endpoint
+    with `container_id` until all (potentially paged) results are received.
+    """
+    entities: List[Any] = []
+    has_more = True
+    paging_token = None
+    filters = filters or {}
+    while has_more and (not max or len(entities) < max):
+        if container_id:
+            resp = list_function(
+                container_id, count=50, paging_token=paging_token, **filters
+            )
+        else:
+            resp = list_function(count=50, paging_token=paging_token, **filters)
+        entities.extend(resp.results)
+        paging_token = resp.metadata.next_paging_token
+        has_more = paging_token is not None
+    return entities
+
+
+T = TypeVar("T")
+
+
+class ListResponseMetadata(BaseModel):
+    total: Optional[int] = None
+    next_paging_token: Optional[str] = None
+
+
+class ListResponse(BaseModel, Generic[T]):
+    results: List[T]
+    metadata: ListResponseMetadata
+
+
+# Must be the `<Entity>Query` Anyscale model. It cannot be a dict.
+QueryObj = TypeVar("QueryObj", bound=object)
+
+
+def search_entities(
+    search_function: Callable[[QueryObj], ListResponse[T]],
+    query_obj: QueryObj,
+    max_to_return: int = 1000,
+) -> List[T]:
+    """Automatically paginate to retrieve all search results up to `max_to_return`
+    (default = 1000).
+
+    Works for POST endpoints where a single query object is passed
+    (in the request body) to the endpoint.
+    """
+    entities: List[T] = []
+    has_more = True
+    count = 0
+
+    while has_more and count < max_to_return:
+        resp = search_function(query_obj)
+        count += len(resp.results)
+        entities += resp.results
+
+        next_paging_token = resp.metadata.next_paging_token
+        query_obj.paging.paging_token = next_paging_token  # type: ignore
+        has_more = next_paging_token is not None
+    if count > max_to_return:
+        count = max_to_return
+        entities = entities[:max_to_return]
+
+    return entities

anyscale/utils/deprecation_util.py

@@ -0,0 +1,32 @@
+from functools import wraps
+from typing import Any, Callable, Optional
+import warnings
+
+
+def deprecated(message: Optional[str] = None) -> Callable[..., Any]:
+    """This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used.
+
+    From https://stackoverflow.com/a/30253848
+    """
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @wraps(func)
+        def decorated_func(*args, **kwargs) -> Any:
+            warnings.simplefilter("always", DeprecationWarning)  # turn on filter
+            deprecated_message = (
+                message
+                if message
+                else f"Call to deprecated function `{func.__name__}`."
+            )
+            # stacklevel=2 logs at the caller level of func
+            # eg. DeprecationWarning: Call to deprecated function `foo()`
+            #   foo()
+            warnings.warn(deprecated_message, DeprecationWarning, stacklevel=2)
+            warnings.filters = warnings.filters[1:]  # reset filter
+            return func(*args, **kwargs)
+
+        return decorated_func
+
+    return decorator