redis 7.0.0b1__py3-none-any.whl → 7.0.0b3__py3-none-any.whl

redis/asyncio/multidb/failover.py

@@ -0,0 +1,125 @@
+import time
+from abc import ABC, abstractmethod
+
+from redis.asyncio.multidb.database import AsyncDatabase, Databases
+from redis.data_structure import WeightedList
+from redis.multidb.circuit import State as CBState
+from redis.multidb.exception import (
+    NoValidDatabaseException,
+    TemporaryUnavailableException,
+)
+
+DEFAULT_FAILOVER_ATTEMPTS = 10
+DEFAULT_FAILOVER_DELAY = 12
+
+
+class AsyncFailoverStrategy(ABC):
+    @abstractmethod
+    async def database(self) -> AsyncDatabase:
+        """Select the database according to the strategy."""
+        pass
+
+    @abstractmethod
+    def set_databases(self, databases: Databases) -> None:
+        """Set the database strategy operates on."""
+        pass
+
+
+class FailoverStrategyExecutor(ABC):
+    @property
+    @abstractmethod
+    def failover_attempts(self) -> int:
+        """The number of failover attempts."""
+        pass
+
+    @property
+    @abstractmethod
+    def failover_delay(self) -> float:
+        """The delay between failover attempts."""
+        pass
+
+    @property
+    @abstractmethod
+    def strategy(self) -> AsyncFailoverStrategy:
+        """The strategy to execute."""
+        pass
+
+    @abstractmethod
+    async def execute(self) -> AsyncDatabase:
+        """Execute the failover strategy."""
+        pass
+
+
+class WeightBasedFailoverStrategy(AsyncFailoverStrategy):
+    """
+    Failover strategy based on database weights.
+    """
+
+    def __init__(self):
+        self._databases = WeightedList()
+
+    async def database(self) -> AsyncDatabase:
+        for database, _ in self._databases:
+            if database.circuit.state == CBState.CLOSED:
+                return database
+
+        raise NoValidDatabaseException("No valid database available for communication")
+
+    def set_databases(self, databases: Databases) -> None:
+        self._databases = databases
+
+
+class DefaultFailoverStrategyExecutor(FailoverStrategyExecutor):
+    """
+    Executes given failover strategy.
+    """
+
+    def __init__(
+        self,
+        strategy: AsyncFailoverStrategy,
+        failover_attempts: int = DEFAULT_FAILOVER_ATTEMPTS,
+        failover_delay: float = DEFAULT_FAILOVER_DELAY,
+    ):
+        self._strategy = strategy
+        self._failover_attempts = failover_attempts
+        self._failover_delay = failover_delay
+        self._next_attempt_ts: int = 0
+        self._failover_counter: int = 0
+
+    @property
+    def failover_attempts(self) -> int:
+        return self._failover_attempts
+
+    @property
+    def failover_delay(self) -> float:
+        return self._failover_delay
+
+    @property
+    def strategy(self) -> AsyncFailoverStrategy:
+        return self._strategy
+
+    async def execute(self) -> AsyncDatabase:
+        try:
+            database = await self._strategy.database()
+            self._reset()
+            return database
+        except NoValidDatabaseException as e:
+            if self._next_attempt_ts == 0:
+                self._next_attempt_ts = time.time() + self._failover_delay
+                self._failover_counter += 1
+            elif time.time() >= self._next_attempt_ts:
+                self._next_attempt_ts += self._failover_delay
+                self._failover_counter += 1
+
+            if self._failover_counter > self._failover_attempts:
+                self._reset()
+                raise e
+            else:
+                raise TemporaryUnavailableException(
+                    "No database connections currently available. "
+                    "This is a temporary condition - please retry the operation."
+                )
+
+    def _reset(self) -> None:
+        self._next_attempt_ts = 0
+        self._failover_counter = 0

redis/asyncio/multidb/failure_detector.py

@@ -0,0 +1,38 @@
+from abc import ABC, abstractmethod
+
+from redis.multidb.failure_detector import FailureDetector
+
+
+class AsyncFailureDetector(ABC):
+    @abstractmethod
+    async def register_failure(self, exception: Exception, cmd: tuple) -> None:
+        """Register a failure that occurred during command execution."""
+        pass
+
+    @abstractmethod
+    async def register_command_execution(self, cmd: tuple) -> None:
+        """Register a command execution."""
+        pass
+
+    @abstractmethod
+    def set_command_executor(self, command_executor) -> None:
+        """Set the command executor for this failure."""
+        pass
+
+
+class FailureDetectorAsyncWrapper(AsyncFailureDetector):
+    """
+    Async wrapper for the failure detector.
+    """
+
+    def __init__(self, failure_detector: FailureDetector) -> None:
+        self._failure_detector = failure_detector
+
+    async def register_failure(self, exception: Exception, cmd: tuple) -> None:
+        self._failure_detector.register_failure(exception, cmd)
+
+    async def register_command_execution(self, cmd: tuple) -> None:
+        self._failure_detector.register_command_execution(cmd)
+
+    def set_command_executor(self, command_executor) -> None:
+        self._failure_detector.set_command_executor(command_executor)

redis/asyncio/multidb/healthcheck.py

@@ -0,0 +1,292 @@
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import List, Optional, Tuple, Union
+
+from redis.asyncio import Redis
+from redis.asyncio.http.http_client import DEFAULT_TIMEOUT, AsyncHTTPClientWrapper
+from redis.backoff import NoBackoff
+from redis.http.http_client import HttpClient
+from redis.multidb.exception import UnhealthyDatabaseException
+from redis.retry import Retry
+
+DEFAULT_HEALTH_CHECK_PROBES = 3
+DEFAULT_HEALTH_CHECK_INTERVAL = 5
+DEFAULT_HEALTH_CHECK_DELAY = 0.5
+DEFAULT_LAG_AWARE_TOLERANCE = 5000
+
+logger = logging.getLogger(__name__)
+
+
+class HealthCheck(ABC):
+    @abstractmethod
+    async def check_health(self, database) -> bool:
+        """Function to determine the health status."""
+        pass
+
+
+class HealthCheckPolicy(ABC):
+    """
+    Health checks execution policy.
+    """
+
+    @property
+    @abstractmethod
+    def health_check_probes(self) -> int:
+        """Number of probes to execute health checks."""
+        pass
+
+    @property
+    @abstractmethod
+    def health_check_delay(self) -> float:
+        """Delay between health check probes."""
+        pass
+
+    @abstractmethod
+    async def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        """Execute health checks and return database health status."""
+        pass
+
+
+class AbstractHealthCheckPolicy(HealthCheckPolicy):
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        if health_check_probes < 1:
+            raise ValueError("health_check_probes must be greater than 0")
+        self._health_check_probes = health_check_probes
+        self._health_check_delay = health_check_delay
+
+    @property
+    def health_check_probes(self) -> int:
+        return self._health_check_probes
+
+    @property
+    def health_check_delay(self) -> float:
+        return self._health_check_delay
+
+    @abstractmethod
+    async def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        pass
+
+
+class HealthyAllPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if all health check probes are successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    async def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        for health_check in health_checks:
+            for attempt in range(self.health_check_probes):
+                try:
+                    if not await health_check.check_health(database):
+                        return False
+                except Exception as e:
+                    raise UnhealthyDatabaseException("Unhealthy database", database, e)
+
+                if attempt < self.health_check_probes - 1:
+                    await asyncio.sleep(self._health_check_delay)
+        return True
+
+
+class HealthyMajorityPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if a majority of health check probes are successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    async def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        for health_check in health_checks:
+            if self.health_check_probes % 2 == 0:
+                allowed_unsuccessful_probes = self.health_check_probes / 2
+            else:
+                allowed_unsuccessful_probes = (self.health_check_probes + 1) / 2
+
+            for attempt in range(self.health_check_probes):
+                try:
+                    if not await health_check.check_health(database):
+                        allowed_unsuccessful_probes -= 1
+                        if allowed_unsuccessful_probes <= 0:
+                            return False
+                except Exception as e:
+                    allowed_unsuccessful_probes -= 1
+                    if allowed_unsuccessful_probes <= 0:
+                        raise UnhealthyDatabaseException(
+                            "Unhealthy database", database, e
+                        )
+
+                if attempt < self.health_check_probes - 1:
+                    await asyncio.sleep(self._health_check_delay)
+        return True
+
+
+class HealthyAnyPolicy(AbstractHealthCheckPolicy):
+    """
+    Policy that returns True if at least one health check probe is successful.
+    """
+
+    def __init__(self, health_check_probes: int, health_check_delay: float):
+        super().__init__(health_check_probes, health_check_delay)
+
+    async def execute(self, health_checks: List[HealthCheck], database) -> bool:
+        is_healthy = False
+
+        for health_check in health_checks:
+            exception = None
+
+            for attempt in range(self.health_check_probes):
+                try:
+                    if await health_check.check_health(database):
+                        is_healthy = True
+                        break
+                    else:
+                        is_healthy = False
+                except Exception as e:
+                    exception = UnhealthyDatabaseException(
+                        "Unhealthy database", database, e
+                    )
+
+                if attempt < self.health_check_probes - 1:
+                    await asyncio.sleep(self._health_check_delay)
+
+            if not is_healthy and not exception:
+                return is_healthy
+            elif not is_healthy and exception:
+                raise exception
+
+        return is_healthy
+
+
+class HealthCheckPolicies(Enum):
+    HEALTHY_ALL = HealthyAllPolicy
+    HEALTHY_MAJORITY = HealthyMajorityPolicy
+    HEALTHY_ANY = HealthyAnyPolicy
+
+
+DEFAULT_HEALTH_CHECK_POLICY: HealthCheckPolicies = HealthCheckPolicies.HEALTHY_ALL
+
+
+class EchoHealthCheck(HealthCheck):
+    """
+    Health check based on ECHO command.
+    """
+
+    async def check_health(self, database) -> bool:
+        expected_message = ["healthcheck", b"healthcheck"]
+
+        if isinstance(database.client, Redis):
+            actual_message = await database.client.execute_command(
+                "ECHO", "healthcheck"
+            )
+            return actual_message in expected_message
+        else:
+            # For a cluster checks if all nodes are healthy.
+            all_nodes = database.client.get_nodes()
+            for node in all_nodes:
+                actual_message = await node.execute_command("ECHO", "healthcheck")
+
+                if actual_message not in expected_message:
+                    return False
+
+            return True
+
+
+class LagAwareHealthCheck(HealthCheck):
+    """
+    Health check available for Redis Enterprise deployments.
+    Verify via REST API that the database is healthy based on different lags.
+    """
+
+    def __init__(
+        self,
+        rest_api_port: int = 9443,
+        lag_aware_tolerance: int = DEFAULT_LAG_AWARE_TOLERANCE,
+        timeout: float = DEFAULT_TIMEOUT,
+        auth_basic: Optional[Tuple[str, str]] = None,
+        verify_tls: bool = True,
+        # TLS verification (server) options
+        ca_file: Optional[str] = None,
+        ca_path: Optional[str] = None,
+        ca_data: Optional[Union[str, bytes]] = None,
+        # Mutual TLS (client cert) options
+        client_cert_file: Optional[str] = None,
+        client_key_file: Optional[str] = None,
+        client_key_password: Optional[str] = None,
+    ):
+        """
+        Initialize LagAwareHealthCheck with the specified parameters.
+
+        Args:
+            rest_api_port: Port number for Redis Enterprise REST API (default: 9443)
+            lag_aware_tolerance: Tolerance in lag between databases in MS (default: 100)
+            timeout: Request timeout in seconds (default: DEFAULT_TIMEOUT)
+            auth_basic: Tuple of (username, password) for basic authentication
+            verify_tls: Whether to verify TLS certificates (default: True)
+            ca_file: Path to CA certificate file for TLS verification
+            ca_path: Path to CA certificates directory for TLS verification
+            ca_data: CA certificate data as string or bytes
+            client_cert_file: Path to client certificate file for mutual TLS
+            client_key_file: Path to client private key file for mutual TLS
+            client_key_password: Password for encrypted client private key
+        """
+        self._http_client = AsyncHTTPClientWrapper(
+            HttpClient(
+                timeout=timeout,
+                auth_basic=auth_basic,
+                retry=Retry(NoBackoff(), retries=0),
+                verify_tls=verify_tls,
+                ca_file=ca_file,
+                ca_path=ca_path,
+                ca_data=ca_data,
+                client_cert_file=client_cert_file,
+                client_key_file=client_key_file,
+                client_key_password=client_key_password,
+            )
+        )
+        self._rest_api_port = rest_api_port
+        self._lag_aware_tolerance = lag_aware_tolerance
+
+    async def check_health(self, database) -> bool:
+        if database.health_check_url is None:
+            raise ValueError(
+                "Database health check url is not set. Please check DatabaseConfig for the current database."
+            )
+
+        if isinstance(database.client, Redis):
+            db_host = database.client.get_connection_kwargs()["host"]
+        else:
+            db_host = database.client.startup_nodes[0].host
+
+        base_url = f"{database.health_check_url}:{self._rest_api_port}"
+        self._http_client.client.base_url = base_url
+
+        # Find bdb matching to the current database host
+        matching_bdb = None
+        for bdb in await self._http_client.get("/v1/bdbs"):
+            for endpoint in bdb["endpoints"]:
+                if endpoint["dns_name"] == db_host:
+                    matching_bdb = bdb
+                    break
+
+                # In case if the host was set as public IP
+                for addr in endpoint["addr"]:
+                    if addr == db_host:
+                        matching_bdb = bdb
+                        break
+
+        if matching_bdb is None:
+            logger.warning("LagAwareHealthCheck failed: Couldn't find a matching bdb")
+            raise ValueError("Could not find a matching bdb")
+
+        url = (
+            f"/v1/bdbs/{matching_bdb['uid']}/availability"
+            f"?extend_check=lag&availability_lag_tolerance_ms={self._lag_aware_tolerance}"
+        )
+        await self._http_client.get(url, expect_json=False)
+
+        # Status checked in an http client, otherwise HttpError will be raised
+        return True

redis/background.py

@@ -0,0 +1,204 @@
+import asyncio
+import threading
+from typing import Any, Callable, Coroutine
+
+
+class BackgroundScheduler:
+    """
+    Schedules background tasks execution either in separate thread or in the running event loop.
+    """
+
+    def __init__(self):
+        self._next_timer = None
+        self._event_loops = []
+        self._lock = threading.Lock()
+        self._stopped = False
+
+    def __del__(self):
+        self.stop()
+
+    def stop(self):
+        """
+        Stop all scheduled tasks and clean up resources.
+        """
+        with self._lock:
+            if self._stopped:
+                return
+            self._stopped = True
+
+            if self._next_timer:
+                self._next_timer.cancel()
+                self._next_timer = None
+
+            # Stop all event loops
+            for loop in self._event_loops:
+                if loop.is_running():
+                    loop.call_soon_threadsafe(loop.stop)
+
+            self._event_loops.clear()
+
+    def run_once(self, delay: float, callback: Callable, *args):
+        """
+        Runs callable task once after certain delay in seconds.
+        """
+        with self._lock:
+            if self._stopped:
+                return
+
+        # Run loop in a separate thread to unblock main thread.
+        loop = asyncio.new_event_loop()
+
+        with self._lock:
+            self._event_loops.append(loop)
+
+        thread = threading.Thread(
+            target=_start_event_loop_in_thread,
+            args=(loop, self._call_later, delay, callback, *args),
+            daemon=True,
+        )
+        thread.start()
+
+    def run_recurring(self, interval: float, callback: Callable, *args):
+        """
+        Runs recurring callable task with given interval in seconds.
+        """
+        with self._lock:
+            if self._stopped:
+                return
+
+        # Run loop in a separate thread to unblock main thread.
+        loop = asyncio.new_event_loop()
+
+        with self._lock:
+            self._event_loops.append(loop)
+
+        thread = threading.Thread(
+            target=_start_event_loop_in_thread,
+            args=(loop, self._call_later_recurring, interval, callback, *args),
+            daemon=True,
+        )
+        thread.start()
+
+    async def run_recurring_async(
+        self, interval: float, coro: Callable[..., Coroutine[Any, Any, Any]], *args
+    ):
+        """
+        Runs recurring coroutine with given interval in seconds in the current event loop.
+        To be used only from an async context. No additional threads are created.
+        """
+        with self._lock:
+            if self._stopped:
+                return
+
+        loop = asyncio.get_running_loop()
+        wrapped = _async_to_sync_wrapper(loop, coro, *args)
+
+        def tick():
+            with self._lock:
+                if self._stopped:
+                    return
+            # Schedule the coroutine
+            wrapped()
+            # Schedule next tick
+            self._next_timer = loop.call_later(interval, tick)
+
+        # Schedule first tick
+        self._next_timer = loop.call_later(interval, tick)
+
+    def _call_later(
+        self, loop: asyncio.AbstractEventLoop, delay: float, callback: Callable, *args
+    ):
+        with self._lock:
+            if self._stopped:
+                return
+        self._next_timer = loop.call_later(delay, callback, *args)
+
+    def _call_later_recurring(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        interval: float,
+        callback: Callable,
+        *args,
+    ):
+        with self._lock:
+            if self._stopped:
+                return
+        self._call_later(
+            loop, interval, self._execute_recurring, loop, interval, callback, *args
+        )
+
+    def _execute_recurring(
+        self,
+        loop: asyncio.AbstractEventLoop,
+        interval: float,
+        callback: Callable,
+        *args,
+    ):
+        """
+        Executes recurring callable task with given interval in seconds.
+        """
+        with self._lock:
+            if self._stopped:
+                return
+
+        try:
+            callback(*args)
+        except Exception:
+            # Silently ignore exceptions during shutdown
+            pass
+
+        with self._lock:
+            if self._stopped:
+                return
+
+        self._call_later(
+            loop, interval, self._execute_recurring, loop, interval, callback, *args
+        )
+
+
+def _start_event_loop_in_thread(
+    event_loop: asyncio.AbstractEventLoop, call_soon_cb: Callable, *args
+):
+    """
+    Starts event loop in a thread and schedule callback as soon as event loop is ready.
+    Used to be able to schedule tasks using loop.call_later.
+
+    :param event_loop:
+    :return:
+    """
+    asyncio.set_event_loop(event_loop)
+    event_loop.call_soon(call_soon_cb, event_loop, *args)
+    try:
+        event_loop.run_forever()
+    finally:
+        try:
+            # Clean up pending tasks
+            pending = asyncio.all_tasks(event_loop)
+            for task in pending:
+                task.cancel()
+            # Run loop once more to process cancellations
+            event_loop.run_until_complete(
+                asyncio.gather(*pending, return_exceptions=True)
+            )
+        except Exception:
+            pass
+        finally:
+            event_loop.close()
+
+
+def _async_to_sync_wrapper(loop, coro_func, *args, **kwargs):
+    """
+    Wraps an asynchronous function so it can be used with loop.call_later.
+
+    :param loop: The event loop in which the coroutine will be executed.
+    :param coro_func: The coroutine function to wrap.
+    :param args: Positional arguments to pass to the coroutine function.
+    :param kwargs: Keyword arguments to pass to the coroutine function.
+    :return: A regular function suitable for loop.call_later.
+    """
+
+    def wrapped():
+        # Schedule the coroutine in the event loop
+        asyncio.ensure_future(coro_func(*args, **kwargs), loop=loop)
+
+    return wrapped