redis 7.0.0b2__py3-none-any.whl → 7.0.1__py3-none-any.whl

redis/multidb/event.py

@@ -0,0 +1,89 @@
+from typing import List
+
+from redis.client import Redis
+from redis.event import EventListenerInterface, OnCommandsFailEvent
+from redis.multidb.database import SyncDatabase
+from redis.multidb.failure_detector import FailureDetector
+
+
+class ActiveDatabaseChanged:
+    """
+    Event fired when an active database has been changed.
+    """
+
+    def __init__(
+        self,
+        old_database: SyncDatabase,
+        new_database: SyncDatabase,
+        command_executor,
+        **kwargs,
+    ):
+        self._old_database = old_database
+        self._new_database = new_database
+        self._command_executor = command_executor
+        self._kwargs = kwargs
+
+    @property
+    def old_database(self) -> SyncDatabase:
+        return self._old_database
+
+    @property
+    def new_database(self) -> SyncDatabase:
+        return self._new_database
+
+    @property
+    def command_executor(self):
+        return self._command_executor
+
+    @property
+    def kwargs(self):
+        return self._kwargs
+
+
+class ResubscribeOnActiveDatabaseChanged(EventListenerInterface):
+    """
+    Re-subscribe the currently active pub / sub to a new active database.
+    """
+
+    def listen(self, event: ActiveDatabaseChanged):
+        old_pubsub = event.command_executor.active_pubsub
+
+        if old_pubsub is not None:
+            # Re-assign old channels and patterns so they will be automatically subscribed on connection.
+            new_pubsub = event.new_database.client.pubsub(**event.kwargs)
+            new_pubsub.channels = old_pubsub.channels
+            new_pubsub.patterns = old_pubsub.patterns
+            new_pubsub.shard_channels = old_pubsub.shard_channels
+            new_pubsub.on_connect(None)
+            event.command_executor.active_pubsub = new_pubsub
+            old_pubsub.close()
+
+
+class CloseConnectionOnActiveDatabaseChanged(EventListenerInterface):
+    """
+    Close connection to the old active database.
+    """
+
+    def listen(self, event: ActiveDatabaseChanged):
+        event.old_database.client.close()
+
+        if isinstance(event.old_database.client, Redis):
+            event.old_database.client.connection_pool.update_active_connections_for_reconnect()
+            event.old_database.client.connection_pool.disconnect()
+        else:
+            for node in event.old_database.client.nodes_manager.nodes_cache.values():
+                node.redis_connection.connection_pool.update_active_connections_for_reconnect()
+                node.redis_connection.connection_pool.disconnect()
+
+
+class RegisterCommandFailure(EventListenerInterface):
+    """
+    Event listener that registers command failures and passing it to the failure detectors.
+    """
+
+    def __init__(self, failure_detectors: List[FailureDetector]):
+        self._failure_detectors = failure_detectors
+
+    def listen(self, event: OnCommandsFailEvent) -> None:
+        for failure_detector in self._failure_detectors:
+            failure_detector.register_failure(event.exception, event.commands)

redis/multidb/exception.py

@@ -0,0 +1,17 @@
+class NoValidDatabaseException(Exception):
+    pass
+
+
+class UnhealthyDatabaseException(Exception):
+    """Exception raised when a database is unhealthy due to an underlying exception."""
+
+    def __init__(self, message, database, original_exception):
+        super().__init__(message)
+        self.database = database
+        self.original_exception = original_exception
+
+
+class TemporaryUnavailableException(Exception):
+    """Exception raised when all databases in setup are temporary unavailable."""
+
+    pass

redis/multidb/failover.py

@@ -0,0 +1,125 @@
+import time
+from abc import ABC, abstractmethod
+
+from redis.data_structure import WeightedList
+from redis.multidb.circuit import State as CBState
+from redis.multidb.database import Databases, SyncDatabase
+from redis.multidb.exception import (
+    NoValidDatabaseException,
+    TemporaryUnavailableException,
+)
+
+DEFAULT_FAILOVER_ATTEMPTS = 10
+DEFAULT_FAILOVER_DELAY = 12
+
+
+class FailoverStrategy(ABC):
+    @abstractmethod
+    def database(self) -> SyncDatabase:
+        """Select the database according to the strategy."""
+        pass
+
+    @abstractmethod
+    def set_databases(self, databases: Databases) -> None:
+        """Set the database strategy operates on."""
+        pass
+
+
+class FailoverStrategyExecutor(ABC):
+    @property
+    @abstractmethod
+    def failover_attempts(self) -> int:
+        """The number of failover attempts."""
+        pass
+
+    @property
+    @abstractmethod
+    def failover_delay(self) -> float:
+        """The delay between failover attempts."""
+        pass
+
+    @property
+    @abstractmethod
+    def strategy(self) -> FailoverStrategy:
+        """The strategy to execute."""
+        pass
+
+    @abstractmethod
+    def execute(self) -> SyncDatabase:
+        """Execute the failover strategy."""
+        pass
+
+
+class WeightBasedFailoverStrategy(FailoverStrategy):
+    """
+    Failover strategy based on database weights.
+    """
+
+    def __init__(self) -> None:
+        self._databases = WeightedList()
+
+    def database(self) -> SyncDatabase:
+        for database, _ in self._databases:
+            if database.circuit.state == CBState.CLOSED:
+                return database
+
+        raise NoValidDatabaseException("No valid database available for communication")
+
+    def set_databases(self, databases: Databases) -> None:
+        self._databases = databases
+
+
+class DefaultFailoverStrategyExecutor(FailoverStrategyExecutor):
+    """
+    Executes given failover strategy.
+    """
+
+    def __init__(
+        self,
+        strategy: FailoverStrategy,
+        failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS,
+        failover_delay: float = DEFAULT_FAILOVER_DELAY,
+    ):
+        self._strategy = strategy
+        self._failover_attempts = failover_attempts
+        self._failover_delay = failover_delay
+        self._next_attempt_ts: int = 0
+        self._failover_counter: int = 0
+
+    @property
+    def failover_attempts(self) -> int:
+        return self._failover_attempts
+
+    @property
+    def failover_delay(self) -> float:
+        return self._failover_delay
+
+    @property
+    def strategy(self) -> FailoverStrategy:
+        return self._strategy
+
+    def execute(self) -> SyncDatabase:
+        try:
+            database = self._strategy.database()
+            self._reset()
+            return database
+        except NoValidDatabaseException as e:
+            if self._next_attempt_ts == 0:
+                self._next_attempt_ts = time.time() + self._failover_delay
+                self._failover_counter += 1
+            elif time.time() >= self._next_attempt_ts:
+                self._next_attempt_ts += self._failover_delay
+                self._failover_counter += 1
+
+            if self._failover_counter > self._failover_attempts:
+                self._reset()
+                raise e
+            else:
+                raise TemporaryUnavailableException(
+                    "No database connections currently available. "
+                    "This is a temporary condition - please retry the operation."
+                )
+
+    def _reset(self) -> None:
+        self._next_attempt_ts = 0
+        self._failover_counter = 0

redis/multidb/failure_detector.py

@@ -0,0 +1,104 @@
+import math
+import threading
+from abc import ABC, abstractmethod
+from datetime import datetime, timedelta
+from typing import List, Type
+
+from typing_extensions import Optional
+
+from redis.multidb.circuit import State as CBState
+
+DEFAULT_MIN_NUM_FAILURES = 1000
+DEFAULT_FAILURE_RATE_THRESHOLD = 0.1
+DEFAULT_FAILURES_DETECTION_WINDOW = 2
+
+
+class FailureDetector(ABC):
+    @abstractmethod
+    def register_failure(self, exception: Exception, cmd: tuple) -> None:
+        """Register a failure that occurred during command execution."""
+        pass
+
+    @abstractmethod
+    def register_command_execution(self, cmd: tuple) -> None:
+        """Register a command execution."""
+        pass
+
+    @abstractmethod
+    def set_command_executor(self, command_executor) -> None:
+        """Set the command executor for this failure."""
+        pass
+
+
+class CommandFailureDetector(FailureDetector):
+    """
+    Detects a failure based on a threshold of failed commands during a specific period of time.
+    """
+
+    def __init__(
+        self,
+        min_num_failures: int = DEFAULT_MIN_NUM_FAILURES,
+        failure_rate_threshold: float = DEFAULT_FAILURE_RATE_THRESHOLD,
+        failure_detection_window: float = DEFAULT_FAILURES_DETECTION_WINDOW,
+        error_types: Optional[List[Type[Exception]]] = None,
+    ) -> None:
+        """
+        Initialize a new CommandFailureDetector instance.
+
+        Args:
+            min_num_failures: Minimal count of failures required for failover
+            failure_rate_threshold: Percentage of failures required for failover
+            failure_detection_window: Time interval for executing health checks.
+            error_types: Optional list of exception types to trigger failover. If None, all exceptions are counted.
+
+        The detector tracks command failures within a sliding time window. When the number of failures
+        exceeds the threshold within the specified duration, it triggers failure detection.
+        """
+        self._command_executor = None
+        self._min_num_failures = min_num_failures
+        self._failure_rate_threshold = failure_rate_threshold
+        self._failure_detection_window = failure_detection_window
+        self._error_types = error_types
+        self._commands_executed: int = 0
+        self._start_time: datetime = datetime.now()
+        self._end_time: datetime = self._start_time + timedelta(
+            seconds=self._failure_detection_window
+        )
+        self._failures_count: int = 0
+        self._lock = threading.RLock()
+
+    def register_failure(self, exception: Exception, cmd: tuple) -> None:
+        with self._lock:
+            if self._error_types:
+                if type(exception) in self._error_types:
+                    self._failures_count += 1
+            else:
+                self._failures_count += 1
+
+            self._check_threshold()
+
+    def set_command_executor(self, command_executor) -> None:
+        self._command_executor = command_executor
+
+    def register_command_execution(self, cmd: tuple) -> None:
+        with self._lock:
+            if not self._start_time < datetime.now() < self._end_time:
+                self._reset()
+
+            self._commands_executed += 1
+
+    def _check_threshold(self):
+        if self._failures_count >= self._min_num_failures and self._failures_count >= (
+            math.ceil(self._commands_executed * self._failure_rate_threshold)
+        ):
+            self._command_executor.active_database.circuit.state = CBState.OPEN
+            self._reset()
+
+    def _reset(self) -> None:
+        with self._lock:
+            self._start_time = datetime.now()
+            self._end_time = self._start_time + timedelta(
+                seconds=self._failure_detection_window
+            )
+            self._failures_count = 0
+            self._commands_executed = 0

redis/multidb/healthcheck.py

@@ -0,0 +1,282 @@
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from time import sleep
+from typing import List, Optional, Tuple, Union
+
+from redis import Redis
+from redis.backoff import NoBackoff
+from redis.http.http_client import DEFAULT_TIMEOUT, HttpClient
+from redis.multidb.exception import UnhealthyDatabaseException
+from redis.retry import Retry
+
+DEFAULT_HEALTH_CHECK_PROBES = 3
+DEFAULT_HEALTH_CHECK_INTERVAL = 5
+DEFAULT_HEALTH_CHECK_DELAY = 0.5
+DEFAULT_LAG_AWARE_TOLERANCE = 5000
+
+logger = logging.getLogger(__name__)
+
+
+class HealthCheck(ABC):
+    @abstractmethod
+    def check_health(self, database) -> bool:
+        """Function to determine the health status."""
+        pass
+
+
+class HealthCheckPolicy(ABC):
+    """
+    Health checks execution policy.
+    """
+
+    @property
+    @abstractmethod
+    def health_check_probes(self) -> int:
+        """Number of probes to execute health checks."""
+        pass
+
+    @property
+    @abstractmethod
+    def health_check_delay(self) -> float:
+        """Delay between health check probes."""
+        pass
+
+    @abstractmethod
+    def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        """Execute health checks and return database health status."""
+        pass
+
+
+class AbstractHealthCheckPolicy(HealthCheckPolicy):
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        if health_check_probes < 1:
+            raise ValueError("health_check_probes must be greater than 0")
+        self._health_check_probes = health_check_probes
+        self._health_check_delay = health_check_delay
+
+    @property
+    def health_check_probes(self) -> int:
+        return self._health_check_probes
+
+    @property
+    def health_check_delay(self) -> float:
+        return self._health_check_delay
+
+    @abstractmethod
+    def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        pass
+
+
+class HealthyAllPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if all health check probes are successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        for health_check in health_checks:
+            for attempt in range(self.health_check_probes):
+                try:
+                    if not health_check.check_health(database):
+                        return False
+                except Exception as e:
+                    raise UnhealthyDatabaseException("Unhealthy database", database, e)
+
+                if attempt < self.health_check_probes - 1:
+                    sleep(self._health_check_delay)
+        return True
+
+
+class HealthyMajorityPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if a majority of health check probes are successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        for health_check in health_checks:
+            if self.health_check_probes % 2 == 0:
+                allowed_unsuccessful_probes = self.health_check_probes / 2
+            else:
+                allowed_unsuccessful_probes = (self.health_check_probes + 1) / 2
+
+            for attempt in range(self.health_check_probes):
+                try:
+                    if not health_check.check_health(database):
+                        allowed_unsuccessful_probes -= 1
+                        if allowed_unsuccessful_probes <= 0:
+                            return False
+                except Exception as e:
+                    allowed_unsuccessful_probes -= 1
+                    if allowed_unsuccessful_probes <= 0:
+                        raise UnhealthyDatabaseException(
+                            "Unhealthy database", database, e
+                        )
+
+                if attempt < self.health_check_probes - 1:
+                    sleep(self._health_check_delay)
+        return True
+
+
+class HealthyAnyPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if at least one health check probe is successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        is_healthy = False
+
+        for health_check in health_checks:
+            exception = None
+
+            for attempt in range(self.health_check_probes):
+                try:
+                    if health_check.check_health(database):
+                        is_healthy = True
+                        break
+                    else:
+                        is_healthy = False
+                except Exception as e:
+                    exception = UnhealthyDatabaseException(
+                        "Unhealthy database", database, e
+                    )
+
+                if attempt < self.health_check_probes - 1:
+                    sleep(self._health_check_delay)
+
+            if not is_healthy and not exception:
+                return is_healthy
+            elif not is_healthy and exception:
+                raise exception
+
+        return is_healthy
+
+
+class HealthCheckPolicies(Enum):
+    HEALTHY_ALL = HealthyAllPolicy
+    HEALTHY_MAJORITY = HealthyMajorityPolicy
+    HEALTHY_ANY = HealthyAnyPolicy
+
+
+DEFAULT_HEALTH_CHECK_POLICY: HealthCheckPolicies = HealthCheckPolicies.HEALTHY_ALL
+
+
+class PingHealthCheck(HealthCheck):
+    """
+    Health check based on PING command.
+    """
+
+    def check_health(self, database) -> bool:
+        if isinstance(database.client, Redis):
+            return database.client.execute_command("PING")
+        else:
+            # For a cluster checks if all nodes are healthy.
+            all_nodes = database.client.get_nodes()
+            for node in all_nodes:
+                if not node.redis_connection.execute_command("PING"):
+                    return False
+
+            return True
+
+
+class LagAwareHealthCheck(HealthCheck):
+    """
+    Health check available for Redis Enterprise deployments.
+    Verify via REST API that the database is healthy based on different lags.
+    """
+
+    def __init__(
+        self,
+        rest_api_port: int = 9443,
+        lag_aware_tolerance: int = DEFAULT_LAG_AWARE_TOLERANCE,
+        timeout: float = DEFAULT_TIMEOUT,
+        auth_basic: Optional[Tuple[str, str]] = None,
+        verify_tls: bool = True,
+        # TLS verification (server) options
+        ca_file: Optional[str] = None,
+        ca_path: Optional[str] = None,
+        ca_data: Optional[Union[str, bytes]] = None,
+        # Mutual TLS (client cert) options
+        client_cert_file: Optional[str] = None,
+        client_key_file: Optional[str] = None,
+        client_key_password: Optional[str] = None,
+    ):
+        """
+        Initialize LagAwareHealthCheck with the specified parameters.
+
+        Args:
+            rest_api_port: Port number for Redis Enterprise REST API (default: 9443)
+            lag_aware_tolerance: Tolerance in lag between databases in MS (default: 100)
+            timeout: Request timeout in seconds (default: DEFAULT_TIMEOUT)
+            auth_basic: Tuple of (username, password) for basic authentication
+            verify_tls: Whether to verify TLS certificates (default: True)
+            ca_file: Path to CA certificate file for TLS verification
+            ca_path: Path to CA certificates directory for TLS verification
+            ca_data: CA certificate data as string or bytes
+            client_cert_file: Path to client certificate file for mutual TLS
+            client_key_file: Path to client private key file for mutual TLS
+            client_key_password: Password for encrypted client private key
+        """
+        self._http_client = HttpClient(
+            timeout=timeout,
+            auth_basic=auth_basic,
+            retry=Retry(NoBackoff(), retries=0),
+            verify_tls=verify_tls,
+            ca_file=ca_file,
+            ca_path=ca_path,
+            ca_data=ca_data,
+            client_cert_file=client_cert_file,
+            client_key_file=client_key_file,
+            client_key_password=client_key_password,
+        )
+        self._rest_api_port = rest_api_port
+        self._lag_aware_tolerance = lag_aware_tolerance
+
+    def check_health(self, database) -> bool:
+        if database.health_check_url is None:
+            raise ValueError(
+                "Database health check url is not set. Please check DatabaseConfig for the current database."
+            )
+
+        if isinstance(database.client, Redis):
+            db_host = database.client.get_connection_kwargs()["host"]
+        else:
+            db_host = database.client.startup_nodes[0].host
+
+        base_url = f"{database.health_check_url}:{self._rest_api_port}"
+        self._http_client.base_url = base_url
+
+        # Find bdb matching to the current database host
+        matching_bdb = None
+        for bdb in self._http_client.get("/v1/bdbs"):
+            for endpoint in bdb["endpoints"]:
+                if endpoint["dns_name"] == db_host:
+                    matching_bdb = bdb
+                    break
+
+                # In case if the host was set as public IP
+                for addr in endpoint["addr"]:
+                    if addr == db_host:
+                        matching_bdb = bdb
+                        break
+
+        if matching_bdb is None:
+            logger.warning("LagAwareHealthCheck failed: Couldn't find a matching bdb")
+            raise ValueError("Could not find a matching bdb")
+
+        url = (
+            f"/v1/bdbs/{matching_bdb['uid']}/availability"
+            f"?extend_check=lag&availability_lag_tolerance_ms={self._lag_aware_tolerance}"
+        )
+        self._http_client.get(url, expect_json=False)
+
+        # Status checked in an http client, otherwise HttpError will be raised
+        return True

redis/retry.py

@@ -1,7 +1,17 @@
 import abc
 import socket
 from time import sleep
-from typing import TYPE_CHECKING, Any, Callable, Generic, Iterable, Tuple, Type, TypeVar
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Generic,
+    Iterable,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
 
 from redis.exceptions import ConnectionError, TimeoutError
 
@@ -91,6 +101,7 @@ class Retry(AbstractRetry[Exception]):
         self,
         do: Callable[[], T],
         fail: Callable[[Exception], Any],
+        is_retryable: Optional[Callable[[Exception], bool]] = None,
     ) -> T:
         """
         Execute an operation that might fail and returns its result, or
@@ -104,6 +115,8 @@ class Retry(AbstractRetry[Exception]):
             try:
                 return do()
             except self._supported_errors as error:
+                if is_retryable and not is_retryable(error):
+                    raise
                 failures += 1
                 fail(error)
                 if self._retries >= 0 and failures > self._retries:

redis/utils.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 import textwrap
+import warnings
 from collections.abc import Callable
 from contextlib import contextmanager
 from functools import wraps
@@ -312,3 +313,36 @@ def truncate_text(txt, max_length=100):
     return textwrap.shorten(
         text=txt, width=max_length, placeholder="...", break_long_words=True
     )
+
+
+def dummy_fail():
+    """
+    Fake function for a Retry object if you don't need to handle each failure.
+    """
+    pass
+
+
+async def dummy_fail_async():
+    """
+    Async fake function for a Retry object if you don't need to handle each failure.
+    """
+    pass
+
+
+def experimental(cls):
+    """
+    Decorator to mark a class as experimental.
+    """
+    original_init = cls.__init__
+
+    @wraps(original_init)
+    def new_init(self, *args, **kwargs):
+        warnings.warn(
+            f"{cls.__name__} is an experimental and may change or be removed in future versions.",
+            category=UserWarning,
+            stacklevel=2,
+        )
+        original_init(self, *args, **kwargs)
+
+    cls.__init__ = new_init
+    return cls