redis 7.0.0b2__py3-none-any.whl → 7.0.1__py3-none-any.whl

redis/multidb/command_executor.py

@@ -0,0 +1,350 @@
+from abc import ABC, abstractmethod
+from datetime import datetime, timedelta
+from typing import Any, Callable, List, Optional
+
+from redis.client import Pipeline, PubSub, PubSubWorkerThread
+from redis.event import EventDispatcherInterface, OnCommandsFailEvent
+from redis.multidb.circuit import State as CBState
+from redis.multidb.config import DEFAULT_AUTO_FALLBACK_INTERVAL
+from redis.multidb.database import Database, Databases, SyncDatabase
+from redis.multidb.event import (
+    ActiveDatabaseChanged,
+    CloseConnectionOnActiveDatabaseChanged,
+    RegisterCommandFailure,
+    ResubscribeOnActiveDatabaseChanged,
+)
+from redis.multidb.failover import (
+    DEFAULT_FAILOVER_ATTEMPTS,
+    DEFAULT_FAILOVER_DELAY,
+    DefaultFailoverStrategyExecutor,
+    FailoverStrategy,
+    FailoverStrategyExecutor,
+)
+from redis.multidb.failure_detector import FailureDetector
+from redis.retry import Retry
+
+
+class CommandExecutor(ABC):
+    @property
+    @abstractmethod
+    def auto_fallback_interval(self) -> float:
+        """Returns auto-fallback interval."""
+        pass
+
+    @auto_fallback_interval.setter
+    @abstractmethod
+    def auto_fallback_interval(self, auto_fallback_interval: float) -> None:
+        """Sets auto-fallback interval."""
+        pass
+
+
+class BaseCommandExecutor(CommandExecutor):
+    def __init__(
+        self,
+        auto_fallback_interval: float = DEFAULT_AUTO_FALLBACK_INTERVAL,
+    ):
+        self._auto_fallback_interval = auto_fallback_interval
+        self._next_fallback_attempt: datetime
+
+    @property
+    def auto_fallback_interval(self) -> float:
+        return self._auto_fallback_interval
+
+    @auto_fallback_interval.setter
+    def auto_fallback_interval(self, auto_fallback_interval: int) -> None:
+        self._auto_fallback_interval = auto_fallback_interval
+
+    def _schedule_next_fallback(self) -> None:
+        if self._auto_fallback_interval == DEFAULT_AUTO_FALLBACK_INTERVAL:
+            return
+
+        self._next_fallback_attempt = datetime.now() + timedelta(
+            seconds=self._auto_fallback_interval
+        )
+
+
+class SyncCommandExecutor(CommandExecutor):
+    @property
+    @abstractmethod
+    def databases(self) -> Databases:
+        """Returns a list of databases."""
+        pass
+
+    @property
+    @abstractmethod
+    def failure_detectors(self) -> List[FailureDetector]:
+        """Returns a list of failure detectors."""
+        pass
+
+    @abstractmethod
+    def add_failure_detector(self, failure_detector: FailureDetector) -> None:
+        """Adds a new failure detector to the list of failure detectors."""
+        pass
+
+    @property
+    @abstractmethod
+    def active_database(self) -> Optional[Database]:
+        """Returns currently active database."""
+        pass
+
+    @active_database.setter
+    @abstractmethod
+    def active_database(self, database: SyncDatabase) -> None:
+        """Sets the currently active database."""
+        pass
+
+    @property
+    @abstractmethod
+    def active_pubsub(self) -> Optional[PubSub]:
+        """Returns currently active pubsub."""
+        pass
+
+    @active_pubsub.setter
+    @abstractmethod
+    def active_pubsub(self, pubsub: PubSub) -> None:
+        """Sets currently active pubsub."""
+        pass
+
+    @property
+    @abstractmethod
+    def failover_strategy_executor(self) -> FailoverStrategyExecutor:
+        """Returns failover strategy executor."""
+        pass
+
+    @property
+    @abstractmethod
+    def command_retry(self) -> Retry:
+        """Returns command retry object."""
+        pass
+
+    @abstractmethod
+    def pubsub(self, **kwargs):
+        """Initializes a PubSub object on a currently active database"""
+        pass
+
+    @abstractmethod
+    def execute_command(self, *args, **options):
+        """Executes a command and returns the result."""
+        pass
+
+    @abstractmethod
+    def execute_pipeline(self, command_stack: tuple):
+        """Executes a stack of commands in pipeline."""
+        pass
+
+    @abstractmethod
+    def execute_transaction(
+        self, transaction: Callable[[Pipeline], None], *watches, **options
+    ):
+        """Executes a transaction block wrapped in callback."""
+        pass
+
+    @abstractmethod
+    def execute_pubsub_method(self, method_name: str, *args, **kwargs):
+        """Executes a given method on active pub/sub."""
+        pass
+
+    @abstractmethod
+    def execute_pubsub_run(self, sleep_time: float, **kwargs) -> Any:
+        """Executes pub/sub run in a thread."""
+        pass
+
+
+class DefaultCommandExecutor(SyncCommandExecutor, BaseCommandExecutor):
+    def __init__(
+        self,
+        failure_detectors: List[FailureDetector],
+        databases: Databases,
+        command_retry: Retry,
+        failover_strategy: FailoverStrategy,
+        event_dispatcher: EventDispatcherInterface,
+        failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS,
+        failover_delay: float = DEFAULT_FAILOVER_DELAY,
+        auto_fallback_interval: float = DEFAULT_AUTO_FALLBACK_INTERVAL,
+    ):
+        """
+        Initialize the DefaultCommandExecutor instance.
+
+        Args:
+            failure_detectors: List of failure detector instances to monitor database health
+            databases: Collection of available databases to execute commands on
+            command_retry: Retry policy for failed command execution
+            failover_strategy: Strategy for handling database failover
+            event_dispatcher: Interface for dispatching events
+            failover_attempts: Number of failover attempts
+            failover_delay: Delay between failover attempts
+            auto_fallback_interval: Time interval in seconds between attempts to fall back to a primary database
+        """
+        super().__init__(auto_fallback_interval)
+
+        for fd in failure_detectors:
+            fd.set_command_executor(command_executor=self)
+
+        self._databases = databases
+        self._failure_detectors = failure_detectors
+        self._command_retry = command_retry
+        self._failover_strategy_executor = DefaultFailoverStrategyExecutor(
+            failover_strategy, failover_attempts, failover_delay
+        )
+        self._event_dispatcher = event_dispatcher
+        self._active_database: Optional[Database] = None
+        self._active_pubsub: Optional[PubSub] = None
+        self._active_pubsub_kwargs = {}
+        self._setup_event_dispatcher()
+        self._schedule_next_fallback()
+
+    @property
+    def databases(self) -> Databases:
+        return self._databases
+
+    @property
+    def failure_detectors(self) -> List[FailureDetector]:
+        return self._failure_detectors
+
+    def add_failure_detector(self, failure_detector: FailureDetector) -> None:
+        self._failure_detectors.append(failure_detector)
+
+    @property
+    def command_retry(self) -> Retry:
+        return self._command_retry
+
+    @property
+    def active_database(self) -> Optional[SyncDatabase]:
+        return self._active_database
+
+    @active_database.setter
+    def active_database(self, database: SyncDatabase) -> None:
+        old_active = self._active_database
+        self._active_database = database
+
+        if old_active is not None and old_active is not database:
+            self._event_dispatcher.dispatch(
+                ActiveDatabaseChanged(
+                    old_active,
+                    self._active_database,
+                    self,
+                    **self._active_pubsub_kwargs,
+                )
+            )
+
+    @property
+    def active_pubsub(self) -> Optional[PubSub]:
+        return self._active_pubsub
+
+    @active_pubsub.setter
+    def active_pubsub(self, pubsub: PubSub) -> None:
+        self._active_pubsub = pubsub
+
+    @property
+    def failover_strategy_executor(self) -> FailoverStrategyExecutor:
+        return self._failover_strategy_executor
+
+    def execute_command(self, *args, **options):
+        def callback():
+            response = self._active_database.client.execute_command(*args, **options)
+            self._register_command_execution(args)
+            return response
+
+        return self._execute_with_failure_detection(callback, args)
+
+    def execute_pipeline(self, command_stack: tuple):
+        def callback():
+            with self._active_database.client.pipeline() as pipe:
+                for command, options in command_stack:
+                    pipe.execute_command(*command, **options)
+
+                response = pipe.execute()
+                self._register_command_execution(command_stack)
+                return response
+
+        return self._execute_with_failure_detection(callback, command_stack)
+
+    def execute_transaction(
+        self, transaction: Callable[[Pipeline], None], *watches, **options
+    ):
+        def callback():
+            response = self._active_database.client.transaction(
+                transaction, *watches, **options
+            )
+            self._register_command_execution(())
+            return response
+
+        return self._execute_with_failure_detection(callback)
+
+    def pubsub(self, **kwargs):
+        def callback():
+            if self._active_pubsub is None:
+                self._active_pubsub = self._active_database.client.pubsub(**kwargs)
+                self._active_pubsub_kwargs = kwargs
+            return None
+
+        return self._execute_with_failure_detection(callback)
+
+    def execute_pubsub_method(self, method_name: str, *args, **kwargs):
+        def callback():
+            method = getattr(self.active_pubsub, method_name)
+            response = method(*args, **kwargs)
+            self._register_command_execution(args)
+            return response
+
+        return self._execute_with_failure_detection(callback, *args)
+
+    def execute_pubsub_run(self, sleep_time, **kwargs) -> "PubSubWorkerThread":
+        def callback():
+            return self._active_pubsub.run_in_thread(sleep_time, **kwargs)
+
+        return self._execute_with_failure_detection(callback)
+
+    def _execute_with_failure_detection(self, callback: Callable, cmds: tuple = ()):
+        """
+        Execute a commands execution callback with failure detection.
+        """
+
+        def wrapper():
+            # On each retry we need to check active database as it might change.
+            self._check_active_database()
+            return callback()
+
+        return self._command_retry.call_with_retry(
+            lambda: wrapper(),
+            lambda error: self._on_command_fail(error, *cmds),
+        )
+
+    def _on_command_fail(self, error, *args):
+        self._event_dispatcher.dispatch(OnCommandsFailEvent(args, error))
+
+    def _check_active_database(self):
+        """
+        Checks if active a database needs to be updated.
+        """
+        if (
+            self._active_database is None
+            or self._active_database.circuit.state != CBState.CLOSED
+            or (
+                self._auto_fallback_interval != DEFAULT_AUTO_FALLBACK_INTERVAL
+                and self._next_fallback_attempt <= datetime.now()
+            )
+        ):
+            self.active_database = self._failover_strategy_executor.execute()
+            self._schedule_next_fallback()
+
+    def _register_command_execution(self, cmd: tuple):
+        for detector in self._failure_detectors:
+            detector.register_command_execution(cmd)
+
+    def _setup_event_dispatcher(self):
+        """
+        Registers necessary listeners.
+        """
+        failure_listener = RegisterCommandFailure(self._failure_detectors)
+        resubscribe_listener = ResubscribeOnActiveDatabaseChanged()
+        close_connection_listener = CloseConnectionOnActiveDatabaseChanged()
+        self._event_dispatcher.register_listeners(
+            {
+                OnCommandsFailEvent: [failure_listener],
+                ActiveDatabaseChanged: [
+                    close_connection_listener,
+                    resubscribe_listener,
+                ],
+            }
+        )

redis/multidb/config.py

@@ -0,0 +1,207 @@
+from dataclasses import dataclass, field
+from typing import List, Type, Union
+
+import pybreaker
+from typing_extensions import Optional
+
+from redis import ConnectionPool, Redis, RedisCluster
+from redis.backoff import ExponentialWithJitterBackoff, NoBackoff
+from redis.data_structure import WeightedList
+from redis.event import EventDispatcher, EventDispatcherInterface
+from redis.multidb.circuit import (
+    DEFAULT_GRACE_PERIOD,
+    CircuitBreaker,
+    PBCircuitBreakerAdapter,
+)
+from redis.multidb.database import Database, Databases
+from redis.multidb.failover import (
+    DEFAULT_FAILOVER_ATTEMPTS,
+    DEFAULT_FAILOVER_DELAY,
+    FailoverStrategy,
+    WeightBasedFailoverStrategy,
+)
+from redis.multidb.failure_detector import (
+    DEFAULT_FAILURE_RATE_THRESHOLD,
+    DEFAULT_FAILURES_DETECTION_WINDOW,
+    DEFAULT_MIN_NUM_FAILURES,
+    CommandFailureDetector,
+    FailureDetector,
+)
+from redis.multidb.healthcheck import (
+    DEFAULT_HEALTH_CHECK_DELAY,
+    DEFAULT_HEALTH_CHECK_INTERVAL,
+    DEFAULT_HEALTH_CHECK_POLICY,
+    DEFAULT_HEALTH_CHECK_PROBES,
+    HealthCheck,
+    HealthCheckPolicies,
+    PingHealthCheck,
+)
+from redis.retry import Retry
+
+DEFAULT_AUTO_FALLBACK_INTERVAL = 120
+
+
+def default_event_dispatcher() -> EventDispatcherInterface:
+    return EventDispatcher()
+
+
+@dataclass
+class DatabaseConfig:
+    """
+    Dataclass representing the configuration for a database connection.
+
+    This class is used to store configuration settings for a database connection,
+    including client options, connection sourcing details, circuit breaker settings,
+    and cluster-specific properties. It provides a structure for defining these
+    attributes and allows for the creation of customized configurations for various
+    database setups.
+
+    Attributes:
+        weight (float): Weight of the database to define the active one.
+        client_kwargs (dict): Additional parameters for the database client connection.
+        from_url (Optional[str]): Redis URL way of connecting to the database.
+        from_pool (Optional[ConnectionPool]): A pre-configured connection pool to use.
+        circuit (Optional[CircuitBreaker]): Custom circuit breaker implementation.
+        grace_period (float): Grace period after which we need to check if the circuit could be closed again.
+        health_check_url (Optional[str]): URL for health checks. Cluster FQDN is typically used
+            on public Redis Enterprise endpoints.
+
+    Methods:
+        default_circuit_breaker:
+            Generates and returns a default CircuitBreaker instance adapted for use.
+    """
+
+    weight: float = 1.0
+    client_kwargs: dict = field(default_factory=dict)
+    from_url: Optional[str] = None
+    from_pool: Optional[ConnectionPool] = None
+    circuit: Optional[CircuitBreaker] = None
+    grace_period: float = DEFAULT_GRACE_PERIOD
+    health_check_url: Optional[str] = None
+
+    def default_circuit_breaker(self) -> CircuitBreaker:
+        circuit_breaker = pybreaker.CircuitBreaker(reset_timeout=self.grace_period)
+        return PBCircuitBreakerAdapter(circuit_breaker)
+
+
+@dataclass
+class MultiDbConfig:
+    """
+    Configuration class for managing multiple database connections in a resilient and fail-safe manner.
+
+    Attributes:
+        databases_config: A list of database configurations.
+        client_class: The client class used to manage database connections.
+        command_retry: Retry strategy for executing database commands.
+        failure_detectors: Optional list of additional failure detectors for monitoring database failures.
+        min_num_failures: Minimal count of failures required for failover
+        failure_rate_threshold: Percentage of failures required for failover
+        failures_detection_window: Time interval for tracking database failures.
+        health_checks: Optional list of additional health checks performed on databases.
+        health_check_interval: Time interval for executing health checks.
+        health_check_probes: Number of attempts to evaluate the health of a database.
+        health_check_probes_delay: Delay between health check attempts.
+        health_check_policy: Policy for determining database health based on health checks.
+        failover_strategy: Optional strategy for handling database failover scenarios.
+        failover_attempts: Number of retries allowed for failover operations.
+        failover_delay: Delay between failover attempts.
+        auto_fallback_interval: Time interval to trigger automatic fallback.
+        event_dispatcher: Interface for dispatching events related to database operations.
+
+    Methods:
+        databases:
+            Retrieves a collection of database clients managed by weighted configurations.
+            Initializes database clients based on the provided configuration and removes
+            redundant retry objects for lower-level clients to rely on global retry logic.
+
+        default_failure_detectors:
+            Returns the default list of failure detectors used to monitor database failures.
+
+        default_health_checks:
+            Returns the default list of health checks used to monitor database health
+            with specific retry and backoff strategies.
+
+        default_failover_strategy:
+            Provides the default failover strategy used for handling failover scenarios
+            with defined retry and backoff configurations.
+    """
+
+    databases_config: List[DatabaseConfig]
+    client_class: Type[Union[Redis, RedisCluster]] = Redis
+    command_retry: Retry = Retry(
+        backoff=ExponentialWithJitterBackoff(base=1, cap=10), retries=3
+    )
+    failure_detectors: Optional[List[FailureDetector]] = None
+    min_num_failures: int = DEFAULT_MIN_NUM_FAILURES
+    failure_rate_threshold: float = DEFAULT_FAILURE_RATE_THRESHOLD
+    failures_detection_window: float = DEFAULT_FAILURES_DETECTION_WINDOW
+    health_checks: Optional[List[HealthCheck]] = None
+    health_check_interval: float = DEFAULT_HEALTH_CHECK_INTERVAL
+    health_check_probes: int = DEFAULT_HEALTH_CHECK_PROBES
+    health_check_probes_delay: float = DEFAULT_HEALTH_CHECK_DELAY
+    health_check_policy: HealthCheckPolicies = DEFAULT_HEALTH_CHECK_POLICY
+    failover_strategy: Optional[FailoverStrategy] = None
+    failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS
+    failover_delay: float = DEFAULT_FAILOVER_DELAY
+    auto_fallback_interval: float = DEFAULT_AUTO_FALLBACK_INTERVAL
+    event_dispatcher: EventDispatcherInterface = field(
+        default_factory=default_event_dispatcher
+    )
+
+    def databases(self) -> Databases:
+        databases = WeightedList()
+
+        for database_config in self.databases_config:
+            # The retry object is not used in the lower level clients, so we can safely remove it.
+            # We rely on command_retry in terms of global retries.
+            database_config.client_kwargs.update(
+                {"retry": Retry(retries=0, backoff=NoBackoff())}
+            )
+
+            if database_config.from_url:
+                client = self.client_class.from_url(
+                    database_config.from_url, **database_config.client_kwargs
+                )
+            elif database_config.from_pool:
+                database_config.from_pool.set_retry(
+                    Retry(retries=0, backoff=NoBackoff())
+                )
+                client = self.client_class.from_pool(
+                    connection_pool=database_config.from_pool
+                )
+            else:
+                client = self.client_class(**database_config.client_kwargs)
+
+            circuit = (
+                database_config.default_circuit_breaker()
+                if database_config.circuit is None
+                else database_config.circuit
+            )
+            databases.add(
+                Database(
+                    client=client,
+                    circuit=circuit,
+                    weight=database_config.weight,
+                    health_check_url=database_config.health_check_url,
+                ),
+                database_config.weight,
+            )
+
+        return databases
+
+    def default_failure_detectors(self) -> List[FailureDetector]:
+        return [
+            CommandFailureDetector(
+                min_num_failures=self.min_num_failures,
+                failure_rate_threshold=self.failure_rate_threshold,
+                failure_detection_window=self.failures_detection_window,
+            ),
+        ]
+
+    def default_health_checks(self) -> List[HealthCheck]:
+        return [
+            PingHealthCheck(),
+        ]
+
+    def default_failover_strategy(self) -> FailoverStrategy:
+        return WeightBasedFailoverStrategy()

redis/multidb/database.py

@@ -0,0 +1,130 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+import redis
+from redis import RedisCluster
+from redis.data_structure import WeightedList
+from redis.multidb.circuit import CircuitBreaker
+from redis.typing import Number
+
+
+class AbstractDatabase(ABC):
+    @property
+    @abstractmethod
+    def weight(self) -> float:
+        """The weight of this database in compare to others. Used to determine the database failover to."""
+        pass
+
+    @weight.setter
+    @abstractmethod
+    def weight(self, weight: float):
+        """Set the weight of this database in compare to others."""
+        pass
+
+    @property
+    @abstractmethod
+    def health_check_url(self) -> Optional[str]:
+        """Health check URL associated with the current database."""
+        pass
+
+    @health_check_url.setter
+    @abstractmethod
+    def health_check_url(self, health_check_url: Optional[str]):
+        """Set the health check URL associated with the current database."""
+        pass
+
+
+class BaseDatabase(AbstractDatabase):
+    def __init__(
+        self,
+        weight: float,
+        health_check_url: Optional[str] = None,
+    ):
+        self._weight = weight
+        self._health_check_url = health_check_url
+
+    @property
+    def weight(self) -> float:
+        return self._weight
+
+    @weight.setter
+    def weight(self, weight: float):
+        self._weight = weight
+
+    @property
+    def health_check_url(self) -> Optional[str]:
+        return self._health_check_url
+
+    @health_check_url.setter
+    def health_check_url(self, health_check_url: Optional[str]):
+        self._health_check_url = health_check_url
+
+
+class SyncDatabase(AbstractDatabase):
+    """Database with an underlying synchronous redis client."""
+
+    @property
+    @abstractmethod
+    def client(self) -> Union[redis.Redis, RedisCluster]:
+        """The underlying redis client."""
+        pass
+
+    @client.setter
+    @abstractmethod
+    def client(self, client: Union[redis.Redis, RedisCluster]):
+        """Set the underlying redis client."""
+        pass
+
+    @property
+    @abstractmethod
+    def circuit(self) -> CircuitBreaker:
+        """Circuit breaker for the current database."""
+        pass
+
+    @circuit.setter
+    @abstractmethod
+    def circuit(self, circuit: CircuitBreaker):
+        """Set the circuit breaker for the current database."""
+        pass
+
+
+Databases = WeightedList[tuple[SyncDatabase, Number]]
+
+
+class Database(BaseDatabase, SyncDatabase):
+    def __init__(
+        self,
+        client: Union[redis.Redis, RedisCluster],
+        circuit: CircuitBreaker,
+        weight: float,
+        health_check_url: Optional[str] = None,
+    ):
+        """
+        Initialize a new Database instance.
+
+        Args:
+            client: Underlying Redis client instance for database operations
+            circuit: Circuit breaker for handling database failures
+            weight: Weight value used for database failover prioritization
+            health_check_url: Health check URL associated with the current database
+        """
+        self._client = client
+        self._cb = circuit
+        self._cb.database = self
+        super().__init__(weight, health_check_url)
+
+    @property
+    def client(self) -> Union[redis.Redis, RedisCluster]:
+        return self._client
+
+    @client.setter
+    def client(self, client: Union[redis.Redis, RedisCluster]):
+        self._client = client
+
+    @property
+    def circuit(self) -> CircuitBreaker:
+        return self._cb
+
+    @circuit.setter
+    def circuit(self, circuit: CircuitBreaker):
+        self._cb = circuit