redis 6.4.0__tar.gz → 7.0.0b1__tar.gz

{redis-6.4.0 → redis-7.0.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis
-Version: 6.4.0
+Version: 7.0.0b1
 Summary: Python client for Redis database and key-value store
 Project-URL: Changes, https://github.com/redis/redis-py/releases
 Project-URL: Code, https://github.com/redis/redis-py

{redis-6.4.0 → redis-7.0.0b1}/redis/__init__.py

@@ -46,7 +46,7 @@ def int_or_str(value):
         return value
 
 
-__version__ = "6.4.0"
+__version__ = "7.0.0b1"
 VERSION = tuple(map(int_or_str, __version__.split(".")))
 
 

{redis-6.4.0 → redis-7.0.0b1}/redis/_parsers/base.py

@@ -1,7 +1,17 @@
+import logging
 import sys
 from abc import ABC
 from asyncio import IncompleteReadError, StreamReader, TimeoutError
-from typing import Callable, List, Optional, Protocol, Union
+from typing import Awaitable, Callable, List, Optional, Protocol, Union
+
+from redis.maintenance_events import (
+    MaintenanceEvent,
+    NodeFailedOverEvent,
+    NodeFailingOverEvent,
+    NodeMigratedEvent,
+    NodeMigratingEvent,
+    NodeMovingEvent,
+)
 
 if sys.version_info.major >= 3 and sys.version_info.minor >= 11:
     from asyncio import timeout as async_timeout
@@ -50,6 +60,8 @@ NO_AUTH_SET_ERROR = {
     "Client sent AUTH, but no password is set": AuthenticationError,
 }
 
+logger = logging.getLogger(__name__)
+
 
 class BaseParser(ABC):
     EXCEPTION_CLASSES = {
@@ -158,7 +170,75 @@ class AsyncBaseParser(BaseParser):
         raise NotImplementedError()
 
 
-_INVALIDATION_MESSAGE = [b"invalidate", "invalidate"]
+class MaintenanceNotificationsParser:
+    """Protocol defining maintenance push notification parsing functionality"""
+
+    @staticmethod
+    def parse_maintenance_start_msg(response, notification_type):
+        # Expected message format is: <event_type> <seq_number> <time>
+        id = response[1]
+        ttl = response[2]
+        return notification_type(id, ttl)
+
+    @staticmethod
+    def parse_maintenance_completed_msg(response, notification_type):
+        # Expected message format is: <event_type> <seq_number>
+        id = response[1]
+        return notification_type(id)
+
+    @staticmethod
+    def parse_moving_msg(response):
+        # Expected message format is: MOVING <seq_number> <time> <endpoint>
+        id = response[1]
+        ttl = response[2]
+        if response[3] is None:
+            host, port = None, None
+        else:
+            value = response[3]
+            if isinstance(value, bytes):
+                value = value.decode()
+            host, port = value.split(":")
+            port = int(port) if port is not None else None
+
+        return NodeMovingEvent(id, host, port, ttl)
+
+
+_INVALIDATION_MESSAGE = "invalidate"
+_MOVING_MESSAGE = "MOVING"
+_MIGRATING_MESSAGE = "MIGRATING"
+_MIGRATED_MESSAGE = "MIGRATED"
+_FAILING_OVER_MESSAGE = "FAILING_OVER"
+_FAILED_OVER_MESSAGE = "FAILED_OVER"
+
+_MAINTENANCE_MESSAGES = (
+    _MIGRATING_MESSAGE,
+    _MIGRATED_MESSAGE,
+    _FAILING_OVER_MESSAGE,
+    _FAILED_OVER_MESSAGE,
+)
+
+MSG_TYPE_TO_EVENT_PARSER_MAPPING: dict[str, tuple[type[MaintenanceEvent], Callable]] = {
+    _MIGRATING_MESSAGE: (
+        NodeMigratingEvent,
+        MaintenanceNotificationsParser.parse_maintenance_start_msg,
+    ),
+    _MIGRATED_MESSAGE: (
+        NodeMigratedEvent,
+        MaintenanceNotificationsParser.parse_maintenance_completed_msg,
+    ),
+    _FAILING_OVER_MESSAGE: (
+        NodeFailingOverEvent,
+        MaintenanceNotificationsParser.parse_maintenance_start_msg,
+    ),
+    _FAILED_OVER_MESSAGE: (
+        NodeFailedOverEvent,
+        MaintenanceNotificationsParser.parse_maintenance_completed_msg,
+    ),
+    _MOVING_MESSAGE: (
+        NodeMovingEvent,
+        MaintenanceNotificationsParser.parse_moving_msg,
+    ),
+}
 
 
 class PushNotificationsParser(Protocol):
@@ -166,16 +246,51 @@ class PushNotificationsParser(Protocol):
 
     pubsub_push_handler_func: Callable
     invalidation_push_handler_func: Optional[Callable] = None
+    node_moving_push_handler_func: Optional[Callable] = None
+    maintenance_push_handler_func: Optional[Callable] = None
 
     def handle_pubsub_push_response(self, response):
         """Handle pubsub push responses"""
         raise NotImplementedError()
 
     def handle_push_response(self, response, **kwargs):
-        if response[0] not in _INVALIDATION_MESSAGE:
+        msg_type = response[0]
+        if isinstance(msg_type, bytes):
+            msg_type = msg_type.decode()
+
+        if msg_type not in (
+            _INVALIDATION_MESSAGE,
+            *_MAINTENANCE_MESSAGES,
+            _MOVING_MESSAGE,
+        ):
             return self.pubsub_push_handler_func(response)
-        if self.invalidation_push_handler_func:
-            return self.invalidation_push_handler_func(response)
+
+        try:
+            if (
+                msg_type == _INVALIDATION_MESSAGE
+                and self.invalidation_push_handler_func
+            ):
+                return self.invalidation_push_handler_func(response)
+
+            if msg_type == _MOVING_MESSAGE and self.node_moving_push_handler_func:
+                parser_function = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][1]
+
+                notification = parser_function(response)
+                return self.node_moving_push_handler_func(notification)
+
+            if msg_type in _MAINTENANCE_MESSAGES and self.maintenance_push_handler_func:
+                parser_function = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][1]
+                notification_type = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][0]
+                notification = parser_function(response, notification_type)
+
+                if notification is not None:
+                    return self.maintenance_push_handler_func(notification)
+        except Exception as e:
+            logger.error(
+                "Error handling {} message ({}): {}".format(msg_type, response, e)
+            )
+
+        return None
 
     def set_pubsub_push_handler(self, pubsub_push_handler_func):
         self.pubsub_push_handler_func = pubsub_push_handler_func
@@ -183,12 +298,20 @@ class PushNotificationsParser(Protocol):
     def set_invalidation_push_handler(self, invalidation_push_handler_func):
         self.invalidation_push_handler_func = invalidation_push_handler_func
 
+    def set_node_moving_push_handler(self, node_moving_push_handler_func):
+        self.node_moving_push_handler_func = node_moving_push_handler_func
+
+    def set_maintenance_push_handler(self, maintenance_push_handler_func):
+        self.maintenance_push_handler_func = maintenance_push_handler_func
+
 
 class AsyncPushNotificationsParser(Protocol):
     """Protocol defining async RESP3-specific parsing functionality"""
 
     pubsub_push_handler_func: Callable
     invalidation_push_handler_func: Optional[Callable] = None
+    node_moving_push_handler_func: Optional[Callable[..., Awaitable[None]]] = None
+    maintenance_push_handler_func: Optional[Callable[..., Awaitable[None]]] = None
 
     async def handle_pubsub_push_response(self, response):
         """Handle pubsub push responses asynchronously"""
@@ -196,10 +319,46 @@ class AsyncPushNotificationsParser(Protocol):
 
     async def handle_push_response(self, response, **kwargs):
         """Handle push responses asynchronously"""
-        if response[0] not in _INVALIDATION_MESSAGE:
+
+        msg_type = response[0]
+        if isinstance(msg_type, bytes):
+            msg_type = msg_type.decode()
+
+        if msg_type not in (
+            _INVALIDATION_MESSAGE,
+            *_MAINTENANCE_MESSAGES,
+            _MOVING_MESSAGE,
+        ):
             return await self.pubsub_push_handler_func(response)
-        if self.invalidation_push_handler_func:
-            return await self.invalidation_push_handler_func(response)
+
+        try:
+            if (
+                msg_type == _INVALIDATION_MESSAGE
+                and self.invalidation_push_handler_func
+            ):
+                return await self.invalidation_push_handler_func(response)
+
+            if isinstance(msg_type, bytes):
+                msg_type = msg_type.decode()
+
+            if msg_type == _MOVING_MESSAGE and self.node_moving_push_handler_func:
+                parser_function = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][1]
+                notification = parser_function(response)
+                return await self.node_moving_push_handler_func(notification)
+
+            if msg_type in _MAINTENANCE_MESSAGES and self.maintenance_push_handler_func:
+                parser_function = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][1]
+                notification_type = MSG_TYPE_TO_EVENT_PARSER_MAPPING[msg_type][0]
+                notification = parser_function(response, notification_type)
+
+                if notification is not None:
+                    return await self.maintenance_push_handler_func(notification)
+        except Exception as e:
+            logger.error(
+                "Error handling {} message ({}): {}".format(msg_type, response, e)
+            )
+
+        return None
 
     def set_pubsub_push_handler(self, pubsub_push_handler_func):
         """Set the pubsub push handler function"""
@@ -209,6 +368,12 @@ class AsyncPushNotificationsParser(Protocol):
         """Set the invalidation push handler function"""
         self.invalidation_push_handler_func = invalidation_push_handler_func
 
+    def set_node_moving_push_handler(self, node_moving_push_handler_func):
+        self.node_moving_push_handler_func = node_moving_push_handler_func
+
+    def set_maintenance_push_handler(self, maintenance_push_handler_func):
+        self.maintenance_push_handler_func = maintenance_push_handler_func
+
 
 class _AsyncRESPBase(AsyncBaseParser):
     """Base class for async resp parsing"""

{redis-6.4.0 → redis-7.0.0b1}/redis/_parsers/hiredis.py

@@ -47,6 +47,8 @@ class _HiredisParser(BaseParser, PushNotificationsParser):
         self.socket_read_size = socket_read_size
         self._buffer = bytearray(socket_read_size)
         self.pubsub_push_handler_func = self.handle_pubsub_push_response
+        self.node_moving_push_handler_func = None
+        self.maintenance_push_handler_func = None
         self.invalidation_push_handler_func = None
         self._hiredis_PushNotificationType = None
 
@@ -141,12 +143,15 @@ class _HiredisParser(BaseParser, PushNotificationsParser):
                 response, self._hiredis_PushNotificationType
             ):
                 response = self.handle_push_response(response)
-                if not push_request:
-                    return self.read_response(
-                        disable_decoding=disable_decoding, push_request=push_request
-                    )
-                else:
+
+                # if this is a push request return the push response
+                if push_request:
                     return response
+
+                return self.read_response(
+                    disable_decoding=disable_decoding,
+                    push_request=push_request,
+                )
             return response
 
         if disable_decoding:
@@ -169,12 +174,13 @@ class _HiredisParser(BaseParser, PushNotificationsParser):
             response, self._hiredis_PushNotificationType
         ):
             response = self.handle_push_response(response)
-            if not push_request:
-                return self.read_response(
-                    disable_decoding=disable_decoding, push_request=push_request
-                )
-            else:
+            if push_request:
                 return response
+            return self.read_response(
+                disable_decoding=disable_decoding,
+                push_request=push_request,
+            )
+
         elif (
             isinstance(response, list)
             and response

{redis-6.4.0 → redis-7.0.0b1}/redis/_parsers/resp3.py

@@ -18,6 +18,8 @@ class _RESP3Parser(_RESPBase, PushNotificationsParser):
     def __init__(self, socket_read_size):
         super().__init__(socket_read_size)
         self.pubsub_push_handler_func = self.handle_pubsub_push_response
+        self.node_moving_push_handler_func = None
+        self.maintenance_push_handler_func = None
         self.invalidation_push_handler_func = None
 
     def handle_pubsub_push_response(self, response):
@@ -117,17 +119,21 @@ class _RESP3Parser(_RESPBase, PushNotificationsParser):
                 for _ in range(int(response))
             ]
             response = self.handle_push_response(response)
-            if not push_request:
-                return self._read_response(
-                    disable_decoding=disable_decoding, push_request=push_request
-                )
-            else:
+
+            # if this is a push request return the push response
+            if push_request:
                 return response
+
+            return self._read_response(
+                disable_decoding=disable_decoding,
+                push_request=push_request,
+            )
         else:
             raise InvalidResponse(f"Protocol Error: {raw!r}")
 
         if isinstance(response, bytes) and disable_decoding is False:
             response = self.encoder.decode(response)
+
         return response
 
 

{redis-6.4.0 → redis-7.0.0b1}/redis/asyncio/client.py

@@ -387,6 +387,12 @@ class Redis(
         # on a set of redis commands
         self._single_conn_lock = asyncio.Lock()
 
+        # When used as an async context manager, we need to increment and decrement
+        # a usage counter so that we can close the connection pool when no one is
+        # using the client.
+        self._usage_counter = 0
+        self._usage_lock = asyncio.Lock()
+
     def __repr__(self):
         return (
             f"<{self.__class__.__module__}.{self.__class__.__name__}"
@@ -594,10 +600,47 @@ class Redis(
         )
 
     async def __aenter__(self: _RedisT) -> _RedisT:
-        return await self.initialize()
+        """
+        Async context manager entry. Increments a usage counter so that the
+        connection pool is only closed (via aclose()) when no context is using
+        the client.
+        """
+        await self._increment_usage()
+        try:
+            # Initialize the client (i.e. establish connection, etc.)
+            return await self.initialize()
+        except Exception:
+            # If initialization fails, decrement the counter to keep it in sync
+            await self._decrement_usage()
+            raise
+
+    async def _increment_usage(self) -> int:
+        """
+        Helper coroutine to increment the usage counter while holding the lock.
+        Returns the new value of the usage counter.
+        """
+        async with self._usage_lock:
+            self._usage_counter += 1
+            return self._usage_counter
+
+    async def _decrement_usage(self) -> int:
+        """
+        Helper coroutine to decrement the usage counter while holding the lock.
+        Returns the new value of the usage counter.
+        """
+        async with self._usage_lock:
+            self._usage_counter -= 1
+            return self._usage_counter
 
     async def __aexit__(self, exc_type, exc_value, traceback):
-        await self.aclose()
+        """
+        Async context manager exit. Decrements a usage counter. If this is the
+        last exit (counter becomes zero), the client closes its connection pool.
+        """
+        current_usage = await asyncio.shield(self._decrement_usage())
+        if current_usage == 0:
+            # This was the last active context, so disconnect the pool.
+            await asyncio.shield(self.aclose())
 
     _DEL_MESSAGE = "Unclosed Redis client"
 

{redis-6.4.0 → redis-7.0.0b1}/redis/asyncio/cluster.py

@@ -431,6 +431,12 @@ class RedisCluster(AbstractRedis, AbstractRedisCluster, AsyncRedisClusterCommand
         self._initialize = True
         self._lock: Optional[asyncio.Lock] = None
 
+        # When used as an async context manager, we need to increment and decrement
+        # a usage counter so that we can close the connection pool when no one is
+        # using the client.
+        self._usage_counter = 0
+        self._usage_lock = asyncio.Lock()
+
     async def initialize(self) -> "RedisCluster":
         """Get all nodes from startup nodes & creates connections if not initialized."""
         if self._initialize:
@@ -467,10 +473,47 @@ class RedisCluster(AbstractRedis, AbstractRedisCluster, AsyncRedisClusterCommand
         await self.aclose()
 
     async def __aenter__(self) -> "RedisCluster":
-        return await self.initialize()
+        """
+        Async context manager entry. Increments a usage counter so that the
+        connection pool is only closed (via aclose()) when no context is using
+        the client.
+        """
+        await self._increment_usage()
+        try:
+            # Initialize the client (i.e. establish connection, etc.)
+            return await self.initialize()
+        except Exception:
+            # If initialization fails, decrement the counter to keep it in sync
+            await self._decrement_usage()
+            raise
 
-    async def __aexit__(self, exc_type: None, exc_value: None, traceback: None) -> None:
-        await self.aclose()
+    async def _increment_usage(self) -> int:
+        """
+        Helper coroutine to increment the usage counter while holding the lock.
+        Returns the new value of the usage counter.
+        """
+        async with self._usage_lock:
+            self._usage_counter += 1
+            return self._usage_counter
+
+    async def _decrement_usage(self) -> int:
+        """
+        Helper coroutine to decrement the usage counter while holding the lock.
+        Returns the new value of the usage counter.
+        """
+        async with self._usage_lock:
+            self._usage_counter -= 1
+            return self._usage_counter
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        """
+        Async context manager exit. Decrements a usage counter. If this is the
+        last exit (counter becomes zero), the client closes its connection pool.
+        """
+        current_usage = await asyncio.shield(self._decrement_usage())
+        if current_usage == 0:
+            # This was the last active context, so disconnect the pool.
+            await asyncio.shield(self.aclose())
 
     def __await__(self) -> Generator[Any, None, "RedisCluster"]:
         return self.initialize().__await__()

{redis-6.4.0 → redis-7.0.0b1}/redis/cache.py

@@ -50,6 +50,7 @@ class EvictionPolicyInterface(ABC):
         pass
 
     @cache.setter
+    @abstractmethod
     def cache(self, value):
         pass
 

{redis-6.4.0 → redis-7.0.0b1}/redis/client.py

@@ -56,6 +56,10 @@ from redis.exceptions import (
     WatchError,
 )
 from redis.lock import Lock
+from redis.maintenance_events import (
+    MaintenanceEventPoolHandler,
+    MaintenanceEventsConfig,
+)
 from redis.retry import Retry
 from redis.utils import (
     _set_info_logger,
@@ -244,6 +248,7 @@ class Redis(RedisModuleCommands, CoreCommands, SentinelCommands):
         cache: Optional[CacheInterface] = None,
         cache_config: Optional[CacheConfig] = None,
         event_dispatcher: Optional[EventDispatcher] = None,
+        maintenance_events_config: Optional[MaintenanceEventsConfig] = None,
     ) -> None:
         """
         Initialize a new Redis client.
@@ -368,6 +373,23 @@ class Redis(RedisModuleCommands, CoreCommands, SentinelCommands):
         ]:
             raise RedisError("Client caching is only supported with RESP version 3")
 
+        if maintenance_events_config and self.connection_pool.get_protocol() not in [
+            3,
+            "3",
+        ]:
+            raise RedisError(
+                "Push handlers on connection are only supported with RESP version 3"
+            )
+        if maintenance_events_config and maintenance_events_config.enabled:
+            self.maintenance_events_pool_handler = MaintenanceEventPoolHandler(
+                self.connection_pool, maintenance_events_config
+            )
+            self.connection_pool.set_maintenance_events_pool_handler(
+                self.maintenance_events_pool_handler
+            )
+        else:
+            self.maintenance_events_pool_handler = None
+
         self.single_connection_lock = threading.RLock()
         self.connection = None
         self._single_connection_client = single_connection_client
@@ -565,8 +587,15 @@ class Redis(RedisModuleCommands, CoreCommands, SentinelCommands):
         return Monitor(self.connection_pool)
 
     def client(self):
+        maintenance_events_config = (
+            None
+            if self.maintenance_events_pool_handler is None
+            else self.maintenance_events_pool_handler.config
+        )
         return self.__class__(
-            connection_pool=self.connection_pool, single_connection_client=True
+            connection_pool=self.connection_pool,
+            single_connection_client=True,
+            maintenance_events_config=maintenance_events_config,
         )
 
     def __enter__(self):
@@ -635,7 +664,11 @@ class Redis(RedisModuleCommands, CoreCommands, SentinelCommands):
                 ),
                 lambda _: self._close_connection(conn),
             )
+
         finally:
+            if conn and conn.should_reconnect():
+                self._close_connection(conn)
+                conn.connect()
             if self._single_connection_client:
                 self.single_connection_lock.release()
             if not self.connection:
@@ -686,11 +719,7 @@ class Monitor:
         self.connection = self.connection_pool.get_connection()
 
     def __enter__(self):
-        self.connection.send_command("MONITOR")
-        # check that monitor returns 'OK', but don't return it to user
-        response = self.connection.read_response()
-        if not bool_ok(response):
-            raise RedisError(f"MONITOR failed: {response}")
+        self._start_monitor()
         return self
 
     def __exit__(self, *args):
@@ -700,8 +729,13 @@ class Monitor:
     def next_command(self):
         """Parse the response from a monitor command"""
         response = self.connection.read_response()
+
+        if response is None:
+            return None
+
         if isinstance(response, bytes):
             response = self.connection.encoder.decode(response, force=True)
+
         command_time, command_data = response.split(" ", 1)
         m = self.monitor_re.match(command_data)
         db_id, client_info, command = m.groups()
@@ -737,6 +771,14 @@ class Monitor:
         while True:
             yield self.next_command()
 
+    def _start_monitor(self):
+        self.connection.send_command("MONITOR")
+        # check that monitor returns 'OK', but don't return it to user
+        response = self.connection.read_response()
+
+        if not bool_ok(response):
+            raise RedisError(f"MONITOR failed: {response}")
+
 
 class PubSub:
     """
@@ -881,7 +923,7 @@ class PubSub:
         """
         ttl = 10
         conn = self.connection
-        while self.health_check_response_counter > 0 and ttl > 0:
+        while conn and self.health_check_response_counter > 0 and ttl > 0:
             if self._execute(conn, conn.can_read, timeout=conn.socket_timeout):
                 response = self._execute(conn, conn.read_response)
                 if self.is_health_check_response(response):
@@ -911,11 +953,17 @@ class PubSub:
         called by the # connection to resubscribe us to any channels and
         patterns we were previously listening to
         """
-        return conn.retry.call_with_retry(
+
+        if conn.should_reconnect():
+            self._reconnect(conn)
+
+        response = conn.retry.call_with_retry(
             lambda: command(*args, **kwargs),
             lambda _: self._reconnect(conn),
         )
 
+        return response
+
     def parse_response(self, block=True, timeout=0):
         """Parse the response from a publish/subscribe command"""
         conn = self.connection
@@ -1125,6 +1173,7 @@ class PubSub:
                 return None
 
         response = self.parse_response(block=(timeout is None), timeout=timeout)
+
         if response:
             return self.handle_message(response, ignore_subscribe_messages)
         return None
@@ -1148,6 +1197,7 @@ class PubSub:
             return None
         if isinstance(response, bytes):
             response = [b"pong", response] if response != b"PONG" else [b"pong", b""]
+
         message_type = str_if_bytes(response[0])
         if message_type == "pmessage":
             message = {
@@ -1351,6 +1401,7 @@ class Pipeline(Redis):
         # clean up the other instance attributes
         self.watching = False
         self.explicit_transaction = False
+
         # we can safely return the connection to the pool here since we're
         # sure we're no longer WATCHing anything
         if self.connection:
@@ -1510,6 +1561,7 @@ class Pipeline(Redis):
                 if command_name in self.response_callbacks:
                     r = self.response_callbacks[command_name](r, **options)
             data.append(r)
+
         return data
 
     def _execute_pipeline(self, connection, commands, raise_on_error):
@@ -1517,16 +1569,17 @@ class Pipeline(Redis):
         all_cmds = connection.pack_commands([args for args, _ in commands])
         connection.send_packed_command(all_cmds)
 
-        response = []
+        responses = []
         for args, options in commands:
             try:
-                response.append(self.parse_response(connection, args[0], **options))
+                responses.append(self.parse_response(connection, args[0], **options))
             except ResponseError as e:
-                response.append(e)
+                responses.append(e)
 
         if raise_on_error:
-            self.raise_first_error(commands, response)
-        return response
+            self.raise_first_error(commands, responses)
+
+        return responses
 
     def raise_first_error(self, commands, response):
         for i, r in enumerate(response):
@@ -1611,6 +1664,8 @@ class Pipeline(Redis):
                 lambda error: self._disconnect_raise_on_watching(conn, error),
             )
         finally:
+            # in reset() the connection is disconnected before returned to the pool if
+            # it is marked for reconnect.
             self.reset()
 
     def discard(self):