rrq 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

rrq/client.py

@@ -8,6 +8,7 @@ from typing import Any, Optional
 from .job import Job, JobStatus
 from .settings import RRQSettings
 from .store import JobStore
+from .telemetry import get_telemetry
 
 logger = logging.getLogger(__name__)
 
@@ -77,110 +78,119 @@ class RRQClient:
             The created Job object if successfully enqueued, or None if enqueueing was denied
             (e.g., due to a unique key conflict).
         """
-        # Determine job ID and enqueue timestamp
+        # Determine job ID and queue name early for telemetry.
         job_id_to_use = _job_id or str(uuid.uuid4())
-        enqueue_time_utc = datetime.now(timezone.utc)
-
-        # Compute base desired run time and unique lock TTL to cover deferral
-        lock_ttl_seconds = self.settings.default_unique_job_lock_ttl_seconds
-        desired_run_time = enqueue_time_utc
-        if _defer_until is not None:
-            dt = _defer_until
-            if dt.tzinfo is None:
-                dt = dt.replace(tzinfo=timezone.utc)
-            elif dt.tzinfo != timezone.utc:
-                dt = dt.astimezone(timezone.utc)
-            desired_run_time = dt
-            diff = (dt - enqueue_time_utc).total_seconds()
-            if diff > 0:
-                lock_ttl_seconds = max(lock_ttl_seconds, int(diff) + 1)
-        elif _defer_by is not None:
-            defer_secs = max(0, int(_defer_by.total_seconds()))
-            desired_run_time = enqueue_time_utc + timedelta(seconds=defer_secs)
-            lock_ttl_seconds = max(lock_ttl_seconds, defer_secs + 1)
-
-        unique_acquired = False
-        # Handle unique key with deferral if locked
-        unique_acquired = False
-        if _unique_key:
-            remaining_ttl = await self.job_store.get_lock_ttl(_unique_key)
-            if remaining_ttl > 0:
-                desired_run_time = max(
-                    desired_run_time, enqueue_time_utc + timedelta(seconds=remaining_ttl)
-                )
-            else:
-                acquired = await self.job_store.acquire_unique_job_lock(
-                    _unique_key, job_id_to_use, lock_ttl_seconds
-                )
-                if acquired:
-                    unique_acquired = True
-                else:
-                    # Race: lock acquired after our check; defer by remaining TTL
-                    remaining = await self.job_store.get_lock_ttl(_unique_key)
+        queue_name_to_use = _queue_name or self.settings.default_queue_name
+
+        telemetry = get_telemetry()
+        with telemetry.enqueue_span(
+            job_id=job_id_to_use,
+            function_name=function_name,
+            queue_name=queue_name_to_use,
+        ) as trace_context:
+            # Determine enqueue timestamp (after telemetry span starts).
+            enqueue_time_utc = datetime.now(timezone.utc)
+
+            # Compute base desired run time and unique lock TTL to cover deferral
+            lock_ttl_seconds = self.settings.default_unique_job_lock_ttl_seconds
+            desired_run_time = enqueue_time_utc
+            if _defer_until is not None:
+                dt = _defer_until
+                if dt.tzinfo is None:
+                    dt = dt.replace(tzinfo=timezone.utc)
+                elif dt.tzinfo != timezone.utc:
+                    dt = dt.astimezone(timezone.utc)
+                desired_run_time = dt
+                diff = (dt - enqueue_time_utc).total_seconds()
+                if diff > 0:
+                    lock_ttl_seconds = max(lock_ttl_seconds, int(diff) + 1)
+            elif _defer_by is not None:
+                defer_secs = max(0, int(_defer_by.total_seconds()))
+                desired_run_time = enqueue_time_utc + timedelta(seconds=defer_secs)
+                lock_ttl_seconds = max(lock_ttl_seconds, defer_secs + 1)
+
+            # Handle unique key with deferral if locked
+            unique_acquired = False
+            if _unique_key:
+                remaining_ttl = await self.job_store.get_lock_ttl(_unique_key)
+                if remaining_ttl > 0:
                     desired_run_time = max(
                         desired_run_time,
-                        enqueue_time_utc + timedelta(seconds=max(0, int(remaining))),
+                        enqueue_time_utc + timedelta(seconds=remaining_ttl),
                     )
+                else:
+                    acquired = await self.job_store.acquire_unique_job_lock(
+                        _unique_key, job_id_to_use, lock_ttl_seconds
+                    )
+                    if acquired:
+                        unique_acquired = True
+                    else:
+                        # Race: lock acquired after our check; defer by remaining TTL
+                        remaining = await self.job_store.get_lock_ttl(_unique_key)
+                        desired_run_time = max(
+                            desired_run_time,
+                            enqueue_time_utc
+                            + timedelta(seconds=max(0, int(remaining))),
+                        )
+
+            # Create the Job instance with all provided details and defaults
+            job = Job(
+                id=job_id_to_use,
+                function_name=function_name,
+                job_args=list(args),
+                job_kwargs=kwargs,
+                enqueue_time=enqueue_time_utc,
+                status=JobStatus.PENDING,
+                current_retries=0,
+                max_retries=(
+                    _max_retries
+                    if _max_retries is not None
+                    else self.settings.default_max_retries
+                ),
+                job_timeout_seconds=(
+                    _job_timeout_seconds
+                    if _job_timeout_seconds is not None
+                    else self.settings.default_job_timeout_seconds
+                ),
+                result_ttl_seconds=(
+                    _result_ttl_seconds
+                    if _result_ttl_seconds is not None
+                    else self.settings.default_result_ttl_seconds
+                ),
+                job_unique_key=_unique_key,
+                queue_name=queue_name_to_use,  # Store the target queue name
+                trace_context=trace_context,
+            )
 
-        queue_name_to_use = _queue_name or self.settings.default_queue_name
+            # Determine the score for the sorted set (queue)
+            # Score is a millisecond timestamp for when the job should be processed.
+            score_dt = desired_run_time
+
+            # Ensure score_dt is timezone-aware (timezone.utc) if it's naive from user input
+            if score_dt.tzinfo is None:
+                score_dt = score_dt.replace(tzinfo=timezone.utc)
+            elif score_dt.tzinfo != timezone.utc:
+                # Convert to timezone.utc if it's aware but not timezone.utc
+                score_dt = score_dt.astimezone(timezone.utc)
+
+            score_timestamp_ms = int(score_dt.timestamp() * 1000)
+            # Record when the job is next scheduled to run (for deferred execution)
+            job.next_scheduled_run_time = score_dt
+
+            # Save the full job definition and add to queue (ensure unique lock is released on error)
+            try:
+                await self.job_store.save_job_definition(job)
+                await self.job_store.add_job_to_queue(
+                    queue_name_to_use,
+                    job.id,
+                    float(score_timestamp_ms),
+                )
+            except Exception:
+                if unique_acquired and _unique_key is not None:
+                    await self.job_store.release_unique_job_lock(_unique_key)
+                raise
 
-        # Create the Job instance with all provided details and defaults
-        job = Job(
-            id=job_id_to_use,
-            function_name=function_name,
-            job_args=list(args),
-            job_kwargs=kwargs,
-            enqueue_time=enqueue_time_utc,
-            status=JobStatus.PENDING,
-            current_retries=0,
-            max_retries=(
-                _max_retries
-                if _max_retries is not None
-                else self.settings.default_max_retries
-            ),
-            job_timeout_seconds=(
-                _job_timeout_seconds
-                if _job_timeout_seconds is not None
-                else self.settings.default_job_timeout_seconds
-            ),
-            result_ttl_seconds=(
-                _result_ttl_seconds
-                if _result_ttl_seconds is not None
-                else self.settings.default_result_ttl_seconds
-            ),
-            job_unique_key=_unique_key,
-            queue_name=queue_name_to_use,  # Store the target queue name
-        )
-
-        # Determine the score for the sorted set (queue)
-        # Score is a millisecond timestamp for when the job should be processed.
-        score_dt = desired_run_time
-
-        # Ensure score_dt is timezone-aware (timezone.utc) if it's naive from user input
-        if score_dt.tzinfo is None:
-            score_dt = score_dt.replace(tzinfo=timezone.utc)
-        elif score_dt.tzinfo != timezone.utc:
-            # Convert to timezone.utc if it's aware but not timezone.utc
-            score_dt = score_dt.astimezone(timezone.utc)
-
-        score_timestamp_ms = int(score_dt.timestamp() * 1000)
-        # Record when the job is next scheduled to run (for deferred execution)
-        job.next_scheduled_run_time = score_dt
-
-        # Save the full job definition and add to queue (ensure unique lock is released on error)
-        try:
-            await self.job_store.save_job_definition(job)
-            await self.job_store.add_job_to_queue(
-                queue_name_to_use,
-                job.id,
-                float(score_timestamp_ms),
+            logger.debug(
+                f"Enqueued job {job.id} ('{job.function_name}') to queue '{queue_name_to_use}' with score {score_timestamp_ms}"
             )
-        except Exception:
-            if unique_acquired:
-                await self.job_store.release_unique_job_lock(_unique_key)
-            raise
-
-        logger.debug(
-            f"Enqueued job {job.id} ('{job.function_name}') to queue '{queue_name_to_use}' with score {score_timestamp_ms}"
-        )
-        return job
+            return job

rrq/exporters/__init__.py

@@ -0,0 +1 @@
+"""Built-in metrics exporters for RRQ hooks."""

rrq/exporters/prometheus.py

@@ -0,0 +1,90 @@
+"""Prometheus metrics exporter for RRQ hooks.
+
+This exporter is optional and requires `prometheus_client` to be installed.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..hooks import MetricsExporter
+from ..settings import RRQSettings
+
+
+class PrometheusExporter(MetricsExporter):
+    """Exports RRQ metrics via `prometheus_client`."""
+
+    def __init__(self, settings: RRQSettings):
+        super().__init__(settings)
+        try:
+            from prometheus_client import Counter, Gauge, Histogram  # type: ignore[import-not-found]
+        except ImportError as e:  # pragma: no cover
+            raise ImportError(
+                "Prometheus exporter requires `prometheus_client` to be installed."
+            ) from e
+
+        self._counter_cls = Counter
+        self._gauge_cls = Gauge
+        self._histogram_cls = Histogram
+
+        self._counters: dict[tuple[str, tuple[str, ...]], Any] = {}
+        self._gauges: dict[tuple[str, tuple[str, ...]], Any] = {}
+        self._histograms: dict[tuple[str, tuple[str, ...]], Any] = {}
+
+    def _get_metric(
+        self,
+        store: dict[tuple[str, tuple[str, ...]], Any],
+        metric_cls: Any,
+        name: str,
+        labelnames: tuple[str, ...],
+    ) -> Any:
+        key = (name, labelnames)
+        metric = store.get(key)
+        if metric is not None:
+            return metric
+
+        description = name
+        if labelnames:
+            metric = metric_cls(name, description, labelnames=labelnames)
+        else:
+            metric = metric_cls(name, description)
+        store[key] = metric
+        return metric
+
+    @staticmethod
+    def _sorted_labelnames(labels: dict[str, str] | None) -> tuple[str, ...]:
+        if not labels:
+            return ()
+        return tuple(sorted(labels.keys()))
+
+    async def export_counter(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        labelnames = self._sorted_labelnames(labels)
+        counter = self._get_metric(self._counters, self._counter_cls, name, labelnames)
+        if labelnames and labels:
+            counter.labels(**{k: labels[k] for k in labelnames}).inc(value)
+        else:
+            counter.inc(value)
+
+    async def export_gauge(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        labelnames = self._sorted_labelnames(labels)
+        gauge = self._get_metric(self._gauges, self._gauge_cls, name, labelnames)
+        if labelnames and labels:
+            gauge.labels(**{k: labels[k] for k in labelnames}).set(value)
+        else:
+            gauge.set(value)
+
+    async def export_histogram(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        labelnames = self._sorted_labelnames(labels)
+        histogram = self._get_metric(
+            self._histograms, self._histogram_cls, name, labelnames
+        )
+        if labelnames and labels:
+            histogram.labels(**{k: labels[k] for k in labelnames}).observe(value)
+        else:
+            histogram.observe(value)

rrq/exporters/statsd.py

@@ -0,0 +1,60 @@
+"""StatsD metrics exporter for RRQ hooks.
+
+This exporter is optional and requires `statsd` to be installed.
+
+Labels are currently ignored because vanilla StatsD does not support tags.
+"""
+
+from __future__ import annotations
+
+import os
+
+from ..hooks import MetricsExporter
+from ..settings import RRQSettings
+
+
+class StatsdExporter(MetricsExporter):
+    """Exports RRQ metrics via `statsd.StatsClient`."""
+
+    def __init__(
+        self,
+        settings: RRQSettings,
+        *,
+        host: str | None = None,
+        port: int | None = None,
+        prefix: str | None = None,
+    ):
+        super().__init__(settings)
+        try:
+            from statsd import StatsClient  # type: ignore[import-not-found]
+        except ImportError as e:  # pragma: no cover
+            raise ImportError(
+                "StatsD exporter requires `statsd` to be installed."
+            ) from e
+
+        resolved_host = host or os.getenv("RRQ_STATSD_HOST", "localhost")
+        resolved_port = port or int(os.getenv("RRQ_STATSD_PORT", "8125"))
+        resolved_prefix = prefix or os.getenv("RRQ_STATSD_PREFIX", "rrq")
+
+        self._client = StatsClient(
+            host=resolved_host, port=resolved_port, prefix=resolved_prefix
+        )
+
+    async def export_counter(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        _ = labels
+        self._client.incr(name, int(value))
+
+    async def export_gauge(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        _ = labels
+        self._client.gauge(name, value)
+
+    async def export_histogram(
+        self, name: str, value: float, labels: dict[str, str] | None = None
+    ) -> None:
+        _ = labels
+        # StatsD doesn't have a standard histogram type; use timing in milliseconds.
+        self._client.timing(name, int(value * 1000))

rrq/hooks.py

@@ -4,7 +4,8 @@ import asyncio
 import importlib
 import logging
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, List
+from collections.abc import Awaitable, Callable
+from typing import Any, cast
 
 from .job import Job
 from .settings import RRQSettings
@@ -39,7 +40,7 @@ class RRQHook(ABC):
         """Called when a job is being retried"""
         pass
 
-    async def on_worker_started(self, worker_id: str, queues: List[str]) -> None:
+    async def on_worker_started(self, worker_id: str, queues: list[str]) -> None:
         """Called when a worker starts"""
         pass
 
@@ -47,7 +48,9 @@ class RRQHook(ABC):
         """Called when a worker stops"""
         pass
 
-    async def on_worker_heartbeat(self, worker_id: str, health_data: Dict) -> None:
+    async def on_worker_heartbeat(
+        self, worker_id: str, health_data: dict[str, Any]
+    ) -> None:
         """Called on worker heartbeat"""
         pass
 
@@ -55,62 +58,67 @@ class RRQHook(ABC):
 class MetricsExporter(ABC):
     """Base class for metrics exporters"""
 
+    def __init__(self, settings: RRQSettings):
+        self.settings = settings
+
     @abstractmethod
     async def export_counter(
-        self, name: str, value: float, labels: Dict[str, str] = None
+        self, name: str, value: float, labels: dict[str, str] | None = None
     ) -> None:
         """Export a counter metric"""
         pass
 
     @abstractmethod
     async def export_gauge(
-        self, name: str, value: float, labels: Dict[str, str] = None
+        self, name: str, value: float, labels: dict[str, str] | None = None
     ) -> None:
         """Export a gauge metric"""
         pass
 
     @abstractmethod
     async def export_histogram(
-        self, name: str, value: float, labels: Dict[str, str] = None
+        self, name: str, value: float, labels: dict[str, str] | None = None
     ) -> None:
         """Export a histogram metric"""
         pass
 
+    async def close(self) -> None:
+        """Close any exporter resources."""
+        return None
+
 
 class HookManager:
     """Manages hooks and exporters for RRQ"""
 
     def __init__(self, settings: RRQSettings):
         self.settings = settings
-        self.hooks: List[RRQHook] = []
-        self.exporters: Dict[str, MetricsExporter] = {}
+        self.hooks: list[RRQHook] = []
+        self.exporters: dict[str, MetricsExporter] = {}
         self._initialized = False
 
-    async def initialize(self):
+    async def initialize(self) -> None:
         """Initialize hooks and exporters from settings"""
         if self._initialized:
             return
 
         # Load event handlers
-        if hasattr(self.settings, "event_handlers"):
-            for handler_path in self.settings.event_handlers:
-                try:
-                    hook = self._load_hook(handler_path)
-                    self.hooks.append(hook)
-                    logger.info(f"Loaded hook: {handler_path}")
-                except Exception as e:
-                    logger.error(f"Failed to load hook {handler_path}: {e}")
+        for handler_path in self.settings.event_handlers:
+            try:
+                hook = self._load_hook(handler_path)
+                self.hooks.append(hook)
+                logger.info(f"Loaded hook: {handler_path}")
+            except Exception as e:
+                logger.error(f"Failed to load hook {handler_path}: {e}")
 
         # Load metrics exporter
-        if hasattr(self.settings, "metrics_exporter"):
-            exporter_type = self.settings.metrics_exporter
-            if exporter_type:
-                try:
-                    exporter = self._load_exporter(exporter_type)
-                    self.exporters[exporter_type] = exporter
-                    logger.info(f"Loaded metrics exporter: {exporter_type}")
-                except Exception as e:
-                    logger.error(f"Failed to load exporter {exporter_type}: {e}")
+        exporter_type = self.settings.metrics_exporter
+        if exporter_type is not None:
+            try:
+                exporter = self._load_exporter(exporter_type)
+                self.exporters[exporter_type] = exporter
+                logger.info(f"Loaded metrics exporter: {exporter_type}")
+            except Exception as e:
+                logger.error(f"Failed to load exporter {exporter_type}: {e}")
 
         self._initialized = True
 
@@ -120,7 +128,7 @@ class HookManager:
         module = importlib.import_module(module_path)
         hook_class = getattr(module, class_name)
 
-        if not issubclass(hook_class, RRQHook):
+        if not isinstance(hook_class, type) or not issubclass(hook_class, RRQHook):
             raise ValueError(f"{handler_path} is not a subclass of RRQHook")
 
         return hook_class(self.settings)
@@ -136,36 +144,62 @@ class HookManager:
             from .exporters.statsd import StatsdExporter
 
             return StatsdExporter(self.settings)
-        else:
-            # Try to load as custom exporter
-            return self._load_hook(exporter_type)
+        return self._load_custom_exporter(exporter_type)
 
-    async def trigger_event(self, event_name: str, *args, **kwargs):
+    def _load_custom_exporter(self, exporter_path: str) -> MetricsExporter:
+        """Load a metrics exporter from a module path."""
+        module_path, class_name = exporter_path.rsplit(".", 1)
+        module = importlib.import_module(module_path)
+        exporter_class = getattr(module, class_name)
+
+        if not isinstance(exporter_class, type) or not issubclass(
+            exporter_class, MetricsExporter
+        ):
+            raise ValueError(f"{exporter_path} is not a subclass of MetricsExporter")
+
+        return exporter_class(self.settings)
+
+    async def trigger_event(self, event_name: str, *args: Any, **kwargs: Any) -> None:
         """Trigger an event on all hooks"""
         if not self._initialized:
             await self.initialize()
 
         # Run hooks concurrently but catch exceptions
-        tasks = []
+        tasks: list[asyncio.Task[object]] = []
         for hook in self.hooks:
-            if hasattr(hook, event_name):
-                method = getattr(hook, event_name)
-                task = asyncio.create_task(self._safe_call(method, *args, **kwargs))
-                tasks.append(task)
+            method = getattr(hook, event_name, None)
+            if method is None:
+                continue
+
+            task = asyncio.create_task(
+                self._safe_call(
+                    cast(Callable[..., Awaitable[Any]], method), *args, **kwargs
+                )
+            )
+            tasks.append(task)
 
         if tasks:
             await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def _safe_call(self, method: Callable, *args, **kwargs):
+    async def _safe_call(
+        self, method: Callable[..., Awaitable[Any]], *args: Any, **kwargs: Any
+    ) -> None:
         """Safely call a hook method"""
         try:
             await method(*args, **kwargs)
         except Exception as e:
-            logger.error(f"Error in hook {method.__qualname__}: {e}")
+            method_name = getattr(
+                method, "__qualname__", getattr(method, "__name__", "")
+            )
+            logger.error(f"Error in hook {method_name}: {e}")
 
     async def export_metric(
-        self, metric_type: str, name: str, value: float, labels: Dict[str, str] = None
-    ):
+        self,
+        metric_type: str,
+        name: str,
+        value: float,
+        labels: dict[str, str] | None = None,
+    ) -> None:
         """Export a metric to all configured exporters"""
         if not self._initialized:
             await self.initialize()
@@ -181,14 +215,13 @@ class HookManager:
             except Exception as e:
                 logger.error(f"Error exporting metric {name}: {e}")
 
-    async def close(self):
+    async def close(self) -> None:
         """Close all exporters"""
         for exporter in self.exporters.values():
-            if hasattr(exporter, "close"):
-                try:
-                    await exporter.close()
-                except Exception as e:
-                    logger.error(f"Error closing exporter: {e}")
+            try:
+                await exporter.close()
+            except Exception as e:
+                logger.error(f"Error closing exporter: {e}")
 
 
 # Example hook implementation
@@ -210,7 +243,7 @@ class LoggingHook(RRQHook):
     async def on_job_retrying(self, job: Job, attempt: int) -> None:
         logger.warning(f"Job retrying: {job.id} - attempt {attempt}")
 
-    async def on_worker_started(self, worker_id: str, queues: List[str]) -> None:
+    async def on_worker_started(self, worker_id: str, queues: list[str]) -> None:
         logger.info(f"Worker started: {worker_id} on queues {queues}")
 
     async def on_worker_stopped(self, worker_id: str) -> None:

rrq/integrations/__init__.py

@@ -0,0 +1 @@
+"""Telemetry integrations for RRQ (optional dependencies)."""