rrq 0.4.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

rrq/client.py

@@ -2,7 +2,7 @@
 
 import logging
 import uuid
-from datetime import UTC, datetime, timedelta
+from datetime import timezone, datetime, timedelta
 from typing import Any, Optional
 
 from .job import Job, JobStatus
@@ -68,7 +68,7 @@ class RRQClient:
                          Uses a Redis lock with `default_unique_job_lock_ttl_seconds`.
             _max_retries: Maximum number of retries for this specific job. Overrides `RRQSettings.default_max_retries`.
             _job_timeout_seconds: Timeout (in seconds) for this specific job. Overrides `RRQSettings.default_job_timeout_seconds`.
-            _defer_until: A specific datetime (UTC recommended) when the job should become available for processing.
+            _defer_until: A specific datetime (timezone.utc recommended) when the job should become available for processing.
             _defer_by: A timedelta relative to now, specifying when the job should become available.
             _result_ttl_seconds: Time-to-live (in seconds) for the result of this specific job. Overrides `RRQSettings.default_result_ttl_seconds`.
             **kwargs: Keyword arguments to pass to the handler function.
@@ -79,40 +79,48 @@ class RRQClient:
         """
         # Determine job ID and enqueue timestamp
         job_id_to_use = _job_id or str(uuid.uuid4())
-        enqueue_time_utc = datetime.now(UTC)
+        enqueue_time_utc = datetime.now(timezone.utc)
 
-        # Compute unique lock TTL: cover deferral window if any
+        # Compute base desired run time and unique lock TTL to cover deferral
         lock_ttl_seconds = self.settings.default_unique_job_lock_ttl_seconds
-        if _defer_by is not None:
-            # Defer relative to now
-            defer_secs = max(0, int(_defer_by.total_seconds()))
-            lock_ttl_seconds = max(lock_ttl_seconds, defer_secs + 1)
-        elif _defer_until is not None:
-            # Defer until specific datetime
+        desired_run_time = enqueue_time_utc
+        if _defer_until is not None:
             dt = _defer_until
-            # Normalize to UTC
             if dt.tzinfo is None:
-                dt = dt.replace(tzinfo=UTC)
-            elif dt.tzinfo != UTC:
-                dt = dt.astimezone(UTC)
+                dt = dt.replace(tzinfo=timezone.utc)
+            elif dt.tzinfo != timezone.utc:
+                dt = dt.astimezone(timezone.utc)
+            desired_run_time = dt
             diff = (dt - enqueue_time_utc).total_seconds()
             if diff > 0:
                 lock_ttl_seconds = max(lock_ttl_seconds, int(diff) + 1)
+        elif _defer_by is not None:
+            defer_secs = max(0, int(_defer_by.total_seconds()))
+            desired_run_time = enqueue_time_utc + timedelta(seconds=defer_secs)
+            lock_ttl_seconds = max(lock_ttl_seconds, defer_secs + 1)
 
         unique_acquired = False
-        # Acquire unique lock if requested, with TTL covering defer window
+        # Handle unique key with deferral if locked
+        unique_acquired = False
         if _unique_key:
-            lock_acquired = await self.job_store.acquire_unique_job_lock(
-                unique_key=_unique_key,
-                job_id=job_id_to_use,
-                lock_ttl_seconds=lock_ttl_seconds,
-            )
-            if not lock_acquired:
-                logger.info(
-                    f"Job with unique key '{_unique_key}' already active or recently run. Enqueue denied."
+            remaining_ttl = await self.job_store.get_lock_ttl(_unique_key)
+            if remaining_ttl > 0:
+                desired_run_time = max(
+                    desired_run_time, enqueue_time_utc + timedelta(seconds=remaining_ttl)
+                )
+            else:
+                acquired = await self.job_store.acquire_unique_job_lock(
+                    _unique_key, job_id_to_use, lock_ttl_seconds
                 )
-                return None
-            unique_acquired = True
+                if acquired:
+                    unique_acquired = True
+                else:
+                    # Race: lock acquired after our check; defer by remaining TTL
+                    remaining = await self.job_store.get_lock_ttl(_unique_key)
+                    desired_run_time = max(
+                        desired_run_time,
+                        enqueue_time_utc + timedelta(seconds=max(0, int(remaining))),
+                    )
 
         queue_name_to_use = _queue_name or self.settings.default_queue_name
 
@@ -146,18 +154,14 @@ class RRQClient:
 
         # Determine the score for the sorted set (queue)
         # Score is a millisecond timestamp for when the job should be processed.
-        score_dt = enqueue_time_utc  # Default to immediate processing
-        if _defer_until:
-            score_dt = _defer_until
-        elif _defer_by:
-            score_dt = enqueue_time_utc + _defer_by
+        score_dt = desired_run_time
 
-        # Ensure score_dt is timezone-aware (UTC) if it's naive from user input
+        # Ensure score_dt is timezone-aware (timezone.utc) if it's naive from user input
         if score_dt.tzinfo is None:
-            score_dt = score_dt.replace(tzinfo=UTC)
-        elif score_dt.tzinfo != UTC:
-            # Convert to UTC if it's aware but not UTC
-            score_dt = score_dt.astimezone(UTC)
+            score_dt = score_dt.replace(tzinfo=timezone.utc)
+        elif score_dt.tzinfo != timezone.utc:
+            # Convert to timezone.utc if it's aware but not timezone.utc
+            score_dt = score_dt.astimezone(timezone.utc)
 
         score_timestamp_ms = int(score_dt.timestamp() * 1000)
         # Record when the job is next scheduled to run (for deferred execution)

rrq/constants.py

@@ -26,6 +26,15 @@ RETRY_COUNTER_PREFIX: str = (
     "rrq:retry_count:"  # Potentially, if not stored directly in job hash
 )
 
+# Hybrid monitoring optimization keys
+ACTIVE_QUEUES_SET: str = (
+    "rrq:active:queues"  # ZSET: queue_name -> last_activity_timestamp
+)
+ACTIVE_WORKERS_SET: str = (
+    "rrq:active:workers"  # ZSET: worker_id -> last_heartbeat_timestamp
+)
+MONITOR_EVENTS_STREAM: str = "rrq:monitor:events"  # Stream for real-time changes
+
 # Default job settings (can be overridden by RRQSettings or per job)
 DEFAULT_MAX_RETRIES: int = 5
 DEFAULT_JOB_TIMEOUT_SECONDS: int = 300  # 5 minutes
@@ -41,3 +50,4 @@ DEFAULT_POLL_DELAY_SECONDS: float = 0.1
 
 # Default worker ID if not specified
 DEFAULT_WORKER_ID_PREFIX: str = "rrq_worker_"
+CONNECTION_POOL_MAX_CONNECTIONS: int = 20

rrq/cron.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import UTC, datetime, timedelta
+from datetime import timezone, datetime, timedelta
 from typing import Any, Optional, Sequence
 
 from pydantic import BaseModel, Field, PrivateAttr
@@ -42,22 +42,24 @@ def _parse_value(value: str, names: dict[str, int], min_val: int, max_val: int)
     return num
 
 
-def _parse_field(field: str, *, names: dict[str, int] | None, min_val: int, max_val: int) -> Sequence[int]:
+def _parse_field(
+    field: str, *, names: dict[str, int] | None, min_val: int, max_val: int
+) -> Sequence[int]:
     names = names or {}
     if field == "*":
         return list(range(min_val, max_val + 1))
     values: set[int] = set()
-    for part in field.split(','):
+    for part in field.split(","):
         step = 1
-        if '/' in part:
-            base, step_str = part.split('/', 1)
+        if "/" in part:
+            base, step_str = part.split("/", 1)
             step = int(step_str)
         else:
             base = part
         if base == "*":
             start, end = min_val, max_val
-        elif '-' in base:
-            a, b = base.split('-', 1)
+        elif "-" in base:
+            a, b = base.split("-", 1)
             start = _parse_value(a, names, min_val, max_val)
             end = _parse_value(b, names, min_val, max_val)
         else:
@@ -88,13 +90,62 @@ class CronSchedule:
 
     def next_after(self, dt: datetime) -> datetime:
         dt = dt.replace(second=0, microsecond=0) + timedelta(minutes=1)
-        while True:
+
+        # Optimization: limit iterations to prevent infinite loops on edge cases
+        max_iterations = 400 * 24 * 60  # ~400 days worth of minutes
+        iterations = 0
+
+        while iterations < max_iterations:
+            iterations += 1
+
+            # Fast skip to next valid month if current month is invalid
             if dt.month not in self.months:
-                dt += timedelta(minutes=1)
+                # Jump to the first day of the next valid month
+                next_month = (
+                    min(m for m in self.months if m > dt.month)
+                    if any(m > dt.month for m in self.months)
+                    else min(self.months)
+                )
+                if next_month <= dt.month:
+                    # Need to go to next year
+                    dt = dt.replace(
+                        year=dt.year + 1, month=next_month, day=1, hour=0, minute=0
+                    )
+                else:
+                    dt = dt.replace(month=next_month, day=1, hour=0, minute=0)
+                continue
+
+            # Fast skip to next valid hour if current hour is invalid
+            if dt.hour not in self.hours:
+                # Jump to the next valid hour
+                next_hour = (
+                    min(h for h in self.hours if h > dt.hour)
+                    if any(h > dt.hour for h in self.hours)
+                    else min(self.hours)
+                )
+                if next_hour <= dt.hour:
+                    # Need to go to next day
+                    dt = (dt + timedelta(days=1)).replace(hour=next_hour, minute=0)
+                else:
+                    dt = dt.replace(hour=next_hour, minute=0)
                 continue
-            if dt.hour not in self.hours or dt.minute not in self.minutes:
-                dt += timedelta(minutes=1)
+
+            # Fast skip to next valid minute if current minute is invalid
+            if dt.minute not in self.minutes:
+                # Jump to the next valid minute
+                next_minute = (
+                    min(m for m in self.minutes if m > dt.minute)
+                    if any(m > dt.minute for m in self.minutes)
+                    else min(self.minutes)
+                )
+                if next_minute <= dt.minute:
+                    # Need to go to next hour
+                    dt = (dt + timedelta(hours=1)).replace(minute=next_minute)
+                else:
+                    dt = dt.replace(minute=next_minute)
                 continue
+
+            # Check day constraints
             dom_match = dt.day in self.dom
             # Convert Python weekday (Monday=0) to cron weekday (Sunday=0)
             # Python: Mon=0, Tue=1, Wed=2, Thu=3, Fri=4, Sat=5, Sun=6
@@ -102,7 +153,7 @@ class CronSchedule:
             python_weekday = dt.weekday()
             cron_weekday = (python_weekday + 1) % 7
             dow_match = cron_weekday in self.dow
-            
+
             if self.dom_all and self.dow_all:
                 condition = True
             elif self.dom_all:
@@ -114,10 +165,19 @@ class CronSchedule:
             else:
                 # Both constraints specified - use OR logic (standard cron behavior)
                 condition = dom_match or dow_match
+
             if condition:
                 return dt
-            dt += timedelta(minutes=1)
 
+            # If day constraints don't match, skip to next day
+            dt = (dt + timedelta(days=1)).replace(
+                hour=min(self.hours), minute=min(self.minutes)
+            )
+
+        # If we've exceeded max iterations, fall back to a reasonable default
+        raise ValueError(
+            f"Could not find next execution time for cron schedule within {max_iterations} iterations"
+        )
 
 
 class CronJob(BaseModel):
@@ -141,13 +201,13 @@ class CronJob(BaseModel):
 
     def schedule_next(self, now: Optional[datetime] = None) -> None:
         """Compute the next run time strictly after *now*."""
-        now = (now or datetime.now(UTC)).replace(second=0, microsecond=0)
+        now = (now or datetime.now(timezone.utc)).replace(second=0, microsecond=0)
         if self._cron is None:
             self._cron = CronSchedule(self.schedule)
         self.next_run_time = self._cron.next_after(now)
 
     def due(self, now: Optional[datetime] = None) -> bool:
-        now = now or datetime.now(UTC)
+        now = now or datetime.now(timezone.utc)
         if self.next_run_time is None:
             self.schedule_next(now)
         return now >= (self.next_run_time or now)

rrq/hooks.py

@@ -0,0 +1,217 @@
+"""Lightweight hooks system for RRQ monitoring and integrations"""
+
+import asyncio
+import importlib
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, List
+
+from .job import Job
+from .settings import RRQSettings
+
+
+logger = logging.getLogger(__name__)
+
+
+class RRQHook(ABC):
+    """Base class for RRQ hooks"""
+
+    def __init__(self, settings: RRQSettings):
+        self.settings = settings
+
+    async def on_job_enqueued(self, job: Job) -> None:
+        """Called when a job is enqueued"""
+        pass
+
+    async def on_job_started(self, job: Job, worker_id: str) -> None:
+        """Called when a job starts processing"""
+        pass
+
+    async def on_job_completed(self, job: Job, result: Any) -> None:
+        """Called when a job completes successfully"""
+        pass
+
+    async def on_job_failed(self, job: Job, error: Exception) -> None:
+        """Called when a job fails"""
+        pass
+
+    async def on_job_retrying(self, job: Job, attempt: int) -> None:
+        """Called when a job is being retried"""
+        pass
+
+    async def on_worker_started(self, worker_id: str, queues: List[str]) -> None:
+        """Called when a worker starts"""
+        pass
+
+    async def on_worker_stopped(self, worker_id: str) -> None:
+        """Called when a worker stops"""
+        pass
+
+    async def on_worker_heartbeat(self, worker_id: str, health_data: Dict) -> None:
+        """Called on worker heartbeat"""
+        pass
+
+
+class MetricsExporter(ABC):
+    """Base class for metrics exporters"""
+
+    @abstractmethod
+    async def export_counter(
+        self, name: str, value: float, labels: Dict[str, str] = None
+    ) -> None:
+        """Export a counter metric"""
+        pass
+
+    @abstractmethod
+    async def export_gauge(
+        self, name: str, value: float, labels: Dict[str, str] = None
+    ) -> None:
+        """Export a gauge metric"""
+        pass
+
+    @abstractmethod
+    async def export_histogram(
+        self, name: str, value: float, labels: Dict[str, str] = None
+    ) -> None:
+        """Export a histogram metric"""
+        pass
+
+
+class HookManager:
+    """Manages hooks and exporters for RRQ"""
+
+    def __init__(self, settings: RRQSettings):
+        self.settings = settings
+        self.hooks: List[RRQHook] = []
+        self.exporters: Dict[str, MetricsExporter] = {}
+        self._initialized = False
+
+    async def initialize(self):
+        """Initialize hooks and exporters from settings"""
+        if self._initialized:
+            return
+
+        # Load event handlers
+        if hasattr(self.settings, "event_handlers"):
+            for handler_path in self.settings.event_handlers:
+                try:
+                    hook = self._load_hook(handler_path)
+                    self.hooks.append(hook)
+                    logger.info(f"Loaded hook: {handler_path}")
+                except Exception as e:
+                    logger.error(f"Failed to load hook {handler_path}: {e}")
+
+        # Load metrics exporter
+        if hasattr(self.settings, "metrics_exporter"):
+            exporter_type = self.settings.metrics_exporter
+            if exporter_type:
+                try:
+                    exporter = self._load_exporter(exporter_type)
+                    self.exporters[exporter_type] = exporter
+                    logger.info(f"Loaded metrics exporter: {exporter_type}")
+                except Exception as e:
+                    logger.error(f"Failed to load exporter {exporter_type}: {e}")
+
+        self._initialized = True
+
+    def _load_hook(self, handler_path: str) -> RRQHook:
+        """Load a hook from a module path"""
+        module_path, class_name = handler_path.rsplit(".", 1)
+        module = importlib.import_module(module_path)
+        hook_class = getattr(module, class_name)
+
+        if not issubclass(hook_class, RRQHook):
+            raise ValueError(f"{handler_path} is not a subclass of RRQHook")
+
+        return hook_class(self.settings)
+
+    def _load_exporter(self, exporter_type: str) -> MetricsExporter:
+        """Load a metrics exporter"""
+        # Built-in exporters
+        if exporter_type == "prometheus":
+            from .exporters.prometheus import PrometheusExporter
+
+            return PrometheusExporter(self.settings)
+        elif exporter_type == "statsd":
+            from .exporters.statsd import StatsdExporter
+
+            return StatsdExporter(self.settings)
+        else:
+            # Try to load as custom exporter
+            return self._load_hook(exporter_type)
+
+    async def trigger_event(self, event_name: str, *args, **kwargs):
+        """Trigger an event on all hooks"""
+        if not self._initialized:
+            await self.initialize()
+
+        # Run hooks concurrently but catch exceptions
+        tasks = []
+        for hook in self.hooks:
+            if hasattr(hook, event_name):
+                method = getattr(hook, event_name)
+                task = asyncio.create_task(self._safe_call(method, *args, **kwargs))
+                tasks.append(task)
+
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _safe_call(self, method: Callable, *args, **kwargs):
+        """Safely call a hook method"""
+        try:
+            await method(*args, **kwargs)
+        except Exception as e:
+            logger.error(f"Error in hook {method.__qualname__}: {e}")
+
+    async def export_metric(
+        self, metric_type: str, name: str, value: float, labels: Dict[str, str] = None
+    ):
+        """Export a metric to all configured exporters"""
+        if not self._initialized:
+            await self.initialize()
+
+        for exporter in self.exporters.values():
+            try:
+                if metric_type == "counter":
+                    await exporter.export_counter(name, value, labels)
+                elif metric_type == "gauge":
+                    await exporter.export_gauge(name, value, labels)
+                elif metric_type == "histogram":
+                    await exporter.export_histogram(name, value, labels)
+            except Exception as e:
+                logger.error(f"Error exporting metric {name}: {e}")
+
+    async def close(self):
+        """Close all exporters"""
+        for exporter in self.exporters.values():
+            if hasattr(exporter, "close"):
+                try:
+                    await exporter.close()
+                except Exception as e:
+                    logger.error(f"Error closing exporter: {e}")
+
+
+# Example hook implementation
+class LoggingHook(RRQHook):
+    """Example hook that logs all events"""
+
+    async def on_job_enqueued(self, job: Job) -> None:
+        logger.info(f"Job enqueued: {job.id} - {job.function_name}")
+
+    async def on_job_started(self, job: Job, worker_id: str) -> None:
+        logger.info(f"Job started: {job.id} on worker {worker_id}")
+
+    async def on_job_completed(self, job: Job, result: Any) -> None:
+        logger.info(f"Job completed: {job.id}")
+
+    async def on_job_failed(self, job: Job, error: Exception) -> None:
+        logger.error(f"Job failed: {job.id} - {error}")
+
+    async def on_job_retrying(self, job: Job, attempt: int) -> None:
+        logger.warning(f"Job retrying: {job.id} - attempt {attempt}")
+
+    async def on_worker_started(self, worker_id: str, queues: List[str]) -> None:
+        logger.info(f"Worker started: {worker_id} on queues {queues}")
+
+    async def on_worker_stopped(self, worker_id: str) -> None:
+        logger.info(f"Worker stopped: {worker_id}")

rrq/job.py

@@ -3,7 +3,7 @@ including the Job model and JobStatus enumeration.
 """
 
 import uuid
-from datetime import UTC, datetime
+from datetime import timezone, datetime
 from enum import Enum
 from typing import Any, Optional
 
@@ -50,8 +50,8 @@ class Job(BaseModel):
     )
 
     enqueue_time: datetime = Field(
-        default_factory=lambda: datetime.now(UTC),
-        description="Timestamp (UTC) when the job was initially enqueued.",
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="Timestamp (timezone.utc) when the job was initially enqueued.",
     )
 
     status: JobStatus = Field(
@@ -62,7 +62,7 @@ class Job(BaseModel):
     )
     next_scheduled_run_time: Optional[datetime] = Field(
         default=None,
-        description="Timestamp (UTC) when the job is next scheduled to run (for retries/deferrals).",
+        description="Timestamp (timezone.utc) when the job is next scheduled to run (for retries/deferrals).",
     )
 
     # Execution control parameters, can be overridden from worker defaults.
@@ -86,7 +86,7 @@ class Job(BaseModel):
     # Fields populated upon job completion or failure.
     completion_time: Optional[datetime] = Field(
         default=None,
-        description="Timestamp (UTC) when the job finished (completed or failed permanently).",
+        description="Timestamp (timezone.utc) when the job finished (completed or failed permanently).",
     )
     result: Optional[Any] = Field(
         default=None,

rrq/registry.py

@@ -69,6 +69,3 @@ class JobRegistry:
     def clear(self) -> None:
         """Clears all registered handlers from the registry."""
         self._handlers.clear()
-
-
-# Global instance for convenience, though applications might manage their own.

rrq/settings.py

@@ -5,7 +5,7 @@ Settings can be loaded from environment variables (with a prefix of `RRQ_`) or
 from a .env file. Sensible defaults are provided for most settings.
 """
 
-from typing import Awaitable, Callable, Optional
+from typing import Awaitable, Callable, List, Optional
 
 from pydantic import Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -102,6 +102,18 @@ class RRQSettings(BaseSettings):
         default_factory=list,
         description="Optional list of cron job specifications to run periodically.",
     )
+    event_handlers: List[str] = Field(
+        default_factory=list,
+        description="List of module paths to event handler classes that implement RRQHook.",
+    )
+    expected_job_ttl: int = Field(
+        default=30,
+        description="Expected job processing time buffer for locks (in seconds)."
+    )
+    metrics_exporter: Optional[str] = Field(
+        default=None,
+        description="Metrics exporter type ('prometheus', 'statsd') or module path to custom exporter.",
+    )
     model_config = SettingsConfigDict(
         env_prefix="RRQ_",
         extra="ignore",