rrq 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

rrq/integrations/otel.py

@@ -0,0 +1,325 @@
+"""OpenTelemetry telemetry integration for RRQ.
+
+This integration is optional and requires OpenTelemetry packages to be installed
+and configured by the application (exporters, tracer provider, etc.).
+"""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+from ..job import Job
+from ..telemetry import EnqueueSpan, JobSpan, Telemetry, configure
+
+
+def enable(*, service_name: str = "rrq") -> None:
+    """Enable OpenTelemetry tracing for RRQ in the current process."""
+    configure(OtelTelemetry(service_name=service_name))
+
+
+class _OtelEnqueueSpan(EnqueueSpan):
+    def __init__(
+        self,
+        *,
+        tracer: Any,
+        service_name: str,
+        job_id: str,
+        function_name: str,
+        queue_name: str,
+    ) -> None:
+        self._tracer = tracer
+        self._service_name = service_name
+        self._job_id = job_id
+        self._function_name = function_name
+        self._queue_name = queue_name
+        self._span_cm: Optional[AbstractContextManager[Any]] = None
+        self._span = None
+
+    def __enter__(self) -> Optional[dict[str, str]]:
+        from opentelemetry import propagate  # type: ignore[import-not-found]
+        from opentelemetry.trace import SpanKind  # type: ignore[import-not-found]
+
+        self._span_cm = self._tracer.start_as_current_span(
+            "rrq.enqueue", kind=SpanKind.PRODUCER
+        )
+        self._span = self._span_cm.__enter__()
+        _otel_set_common_attributes(
+            self._span,
+            job_id=self._job_id,
+            function_name=self._function_name,
+            queue_name=self._queue_name,
+            service_name=self._service_name,
+            kind="producer",
+        )
+
+        carrier: dict[str, str] = {}
+        try:
+            propagate.inject(carrier)
+        except Exception:
+            carrier = {}
+        return carrier or None
+
+    def __exit__(self, exc_type, exc, tb) -> bool:  # type: ignore[override]
+        if self._span is not None and exc is not None:
+            _otel_record_exception(self._span, exc)
+        try:
+            if self._span_cm is not None:
+                return bool(self._span_cm.__exit__(exc_type, exc, tb))
+            return False
+        finally:
+            self._span_cm = None
+            self._span = None
+
+
+class _OtelJobSpan(JobSpan):
+    def __init__(
+        self,
+        *,
+        tracer: Any,
+        service_name: str,
+        job: Job,
+        worker_id: str,
+        queue_name: str,
+        attempt: int,
+        timeout_seconds: float,
+    ) -> None:
+        self._tracer = tracer
+        self._service_name = service_name
+        self._job = job
+        self._worker_id = worker_id
+        self._queue_name = queue_name
+        self._attempt = attempt
+        self._timeout_seconds = timeout_seconds
+        self._span_cm: Optional[AbstractContextManager[Any]] = None
+        self._span = None
+
+    def __enter__(self) -> "_OtelJobSpan":
+        from opentelemetry import propagate  # type: ignore[import-not-found]
+        from opentelemetry.trace import SpanKind  # type: ignore[import-not-found]
+
+        context = None
+        if self._job.trace_context:
+            try:
+                context = propagate.extract(dict(self._job.trace_context))
+            except Exception:
+                context = None
+
+        if context is not None:
+            self._span_cm = self._tracer.start_as_current_span(
+                "rrq.job", context=context, kind=SpanKind.CONSUMER
+            )
+        else:
+            self._span_cm = self._tracer.start_as_current_span(
+                "rrq.job", kind=SpanKind.CONSUMER
+            )
+        self._span = self._span_cm.__enter__()
+
+        _otel_set_common_attributes(
+            self._span,
+            job_id=self._job.id,
+            function_name=self._job.function_name,
+            queue_name=self._queue_name,
+            service_name=self._service_name,
+            kind="consumer",
+        )
+        try:
+            self._span.set_attribute("rrq.worker_id", self._worker_id)
+            self._span.set_attribute("rrq.attempt", self._attempt)
+            self._span.set_attribute(
+                "rrq.timeout_seconds", float(self._timeout_seconds)
+            )
+            self._span.set_attribute(
+                "rrq.queue_delay_ms", _calculate_queue_delay_ms(self._job)
+            )
+        except Exception:
+            pass
+
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:  # type: ignore[override]
+        if self._span is not None and exc is not None:
+            _otel_record_exception(self._span, exc)
+        try:
+            if self._span_cm is not None:
+                return bool(self._span_cm.__exit__(exc_type, exc, tb))
+            return False
+        finally:
+            self._span_cm = None
+            self._span = None
+
+    def success(self, *, duration_seconds: float) -> None:
+        _otel_set_outcome(self._span, "success", duration_seconds=duration_seconds)
+
+    def retry(
+        self,
+        *,
+        duration_seconds: float,
+        delay_seconds: Optional[float] = None,
+        reason: Optional[str] = None,
+    ) -> None:
+        _otel_set_outcome(
+            self._span,
+            "retry",
+            duration_seconds=duration_seconds,
+            delay_seconds=delay_seconds,
+            reason=reason,
+        )
+
+    def dlq(
+        self,
+        *,
+        duration_seconds: float,
+        reason: Optional[str] = None,
+        error: Optional[BaseException] = None,
+    ) -> None:
+        if self._span is not None and error is not None:
+            _otel_record_exception(self._span, error)
+        _otel_set_outcome(
+            self._span, "dlq", duration_seconds=duration_seconds, reason=reason
+        )
+
+    def timeout(
+        self,
+        *,
+        duration_seconds: float,
+        timeout_seconds: float,
+        error_message: Optional[str] = None,
+    ) -> None:
+        if self._span is not None:
+            try:
+                self._span.set_attribute("rrq.timeout_seconds", float(timeout_seconds))
+                if error_message:
+                    self._span.set_attribute("rrq.error_message", error_message)
+            except Exception:
+                pass
+        _otel_set_outcome(self._span, "timeout", duration_seconds=duration_seconds)
+
+    def cancelled(
+        self, *, duration_seconds: float, reason: Optional[str] = None
+    ) -> None:
+        _otel_set_outcome(
+            self._span, "cancelled", duration_seconds=duration_seconds, reason=reason
+        )
+
+    def close(self) -> None:
+        return
+
+
+class OtelTelemetry(Telemetry):
+    """OpenTelemetry-backed RRQ telemetry (traces + propagation)."""
+
+    enabled: bool = True
+
+    def __init__(self, *, service_name: str) -> None:
+        try:
+            from opentelemetry import trace  # type: ignore[import-not-found]
+        except Exception as e:
+            raise RuntimeError(
+                "OpenTelemetry is not installed; install opentelemetry-api and your exporter."
+            ) from e
+        self._service_name = service_name
+        self._tracer = trace.get_tracer("rrq")
+
+    def enqueue_span(
+        self, *, job_id: str, function_name: str, queue_name: str
+    ) -> EnqueueSpan:
+        return _OtelEnqueueSpan(
+            tracer=self._tracer,
+            service_name=self._service_name,
+            job_id=job_id,
+            function_name=function_name,
+            queue_name=queue_name,
+        )
+
+    def job_span(
+        self,
+        *,
+        job: Job,
+        worker_id: str,
+        queue_name: str,
+        attempt: int,
+        timeout_seconds: float,
+    ) -> JobSpan:
+        return _OtelJobSpan(
+            tracer=self._tracer,
+            service_name=self._service_name,
+            job=job,
+            worker_id=worker_id,
+            queue_name=queue_name,
+            attempt=attempt,
+            timeout_seconds=timeout_seconds,
+        )
+
+
+def _otel_set_common_attributes(
+    span: Any,
+    *,
+    job_id: str,
+    function_name: str,
+    queue_name: str,
+    service_name: str,
+    kind: str,
+) -> None:
+    if span is None:
+        return
+    try:
+        span.set_attribute("service.name", service_name)
+        span.set_attribute("rrq.job_id", job_id)
+        span.set_attribute("rrq.function", function_name)
+        span.set_attribute("rrq.queue", queue_name)
+        span.set_attribute("span.kind", kind)
+        span.set_attribute("messaging.system", "redis")
+        span.set_attribute("messaging.destination.name", queue_name)
+        span.set_attribute("messaging.destination_kind", "queue")
+    except Exception:
+        pass
+
+
+def _otel_set_outcome(
+    span: Any,
+    outcome: str,
+    *,
+    duration_seconds: float,
+    delay_seconds: Optional[float] = None,
+    reason: Optional[str] = None,
+) -> None:
+    if span is None:
+        return
+    try:
+        span.set_attribute("rrq.outcome", outcome)
+        span.set_attribute("rrq.duration_ms", float(duration_seconds) * 1000.0)
+        if delay_seconds is not None:
+            span.set_attribute("rrq.retry_delay_ms", float(delay_seconds) * 1000.0)
+        if reason:
+            span.set_attribute("rrq.reason", reason)
+    except Exception:
+        pass
+
+
+def _otel_record_exception(span: Any, error: BaseException) -> None:
+    if span is None:
+        return
+    try:
+        span.record_exception(error)
+    except Exception:
+        pass
+
+    try:
+        from opentelemetry.trace import Status, StatusCode  # type: ignore[import-not-found]
+
+        span.set_status(Status(StatusCode.ERROR))
+    except Exception:
+        pass
+
+
+def _calculate_queue_delay_ms(job: Job) -> float:
+    scheduled_time = job.next_scheduled_run_time or job.enqueue_time
+    dt = scheduled_time
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    elif dt.tzinfo != timezone.utc:
+        dt = dt.astimezone(timezone.utc)
+    delay_ms = (datetime.now(timezone.utc) - dt).total_seconds() * 1000.0
+    return max(0.0, delay_ms)

rrq/job.py

@@ -105,3 +105,9 @@ class Job(BaseModel):
         default=None,
         description="The name of the Dead Letter Queue this job will be moved to if it fails permanently.",
     )
+
+    # Distributed tracing context carrier (serialized by JobStore).
+    trace_context: Optional[dict[str, str]] = Field(
+        default=None,
+        description="Optional distributed tracing propagation carrier to continue traces from enqueue to execution.",
+    )

rrq/settings.py

@@ -78,7 +78,7 @@ class RRQSettings(BaseSettings):
         default=10,
         description="Default number of concurrent jobs a single worker process can handle.",
     )
-    worker_health_check_interval_seconds: int = Field(
+    worker_health_check_interval_seconds: float = Field(
         default=60,
         description="Interval (in seconds) at which a worker updates its health check status in Redis.",
     )
@@ -108,7 +108,7 @@ class RRQSettings(BaseSettings):
     )
     expected_job_ttl: int = Field(
         default=30,
-        description="Expected job processing time buffer for locks (in seconds)."
+        description="Expected job processing time buffer for locks (in seconds).",
     )
     metrics_exporter: Optional[str] = Field(
         default=None,

rrq/store.py

@@ -102,7 +102,7 @@ class JobStore:
     async def save_job_definition(self, job: Job) -> None:
         """Saves the complete job definition as a Redis hash.
 
-        Handles manual serialization of complex fields (args, kwargs, result).
+        Handles manual serialization of complex fields (args, kwargs, result, trace_context).
 
         Args:
             job: The Job object to save.
@@ -111,7 +111,7 @@ class JobStore:
 
         # Dump model excluding fields handled manually
         job_data_dict = job.model_dump(
-            mode="json", exclude={"job_args", "job_kwargs", "result"}
+            mode="json", exclude={"job_args", "job_kwargs", "result", "trace_context"}
         )
 
         # Manually serialize potentially complex fields to JSON strings
@@ -120,6 +120,9 @@ class JobStore:
             job.job_kwargs if job.job_kwargs is not None else None
         )
         result_json = json.dumps(job.result if job.result is not None else None)
+        trace_context_json = None
+        if job.trace_context is not None:
+            trace_context_json = json.dumps(job.trace_context)
 
         # Combine base fields (converted to string) with manually serialized ones
         final_mapping_for_hset = {
@@ -128,6 +131,8 @@ class JobStore:
         final_mapping_for_hset["job_args"] = job_args_json
         final_mapping_for_hset["job_kwargs"] = job_kwargs_json
         final_mapping_for_hset["result"] = result_json
+        if trace_context_json is not None:
+            final_mapping_for_hset["trace_context"] = trace_context_json
 
         # Ensure ID is present
         if "id" not in final_mapping_for_hset:
@@ -164,10 +169,12 @@ class JobStore:
         job_args_list = None
         job_kwargs_dict = None
         result_obj = None
+        trace_context_obj: Optional[dict[str, str]] = None
 
         job_args_str = job_data_dict_str.pop("job_args", None)
         job_kwargs_str = job_data_dict_str.pop("job_kwargs", None)
         result_str = job_data_dict_str.pop("result", None)
+        trace_context_str = job_data_dict_str.pop("trace_context", None)
 
         if job_args_str and job_args_str.lower() != "null":
             try:
@@ -200,6 +207,19 @@ class JobStore:
                 # If stored via json.dumps, failure here indicates corruption or non-JSON string stored previously.
                 result_obj = None  # Safest fallback is likely None
 
+        if trace_context_str and trace_context_str.lower() != "null":
+            try:
+                parsed = json.loads(trace_context_str)
+                if isinstance(parsed, dict):
+                    trace_context_obj = {
+                        str(k): str(v) for k, v in parsed.items() if v is not None
+                    }
+            except json.JSONDecodeError:
+                logger.error(
+                    f"Failed to JSON decode 'trace_context' for job {job_id} from string: '{trace_context_str}'",
+                    exc_info=True,
+                )
+
         # Validate the remaining dictionary using Pydantic Job model
         try:
             # Pass only the remaining fields to the constructor
@@ -212,6 +232,7 @@ class JobStore:
                 job_kwargs_dict if job_kwargs_dict is not None else {}
             )
             validated_job.result = result_obj
+            validated_job.trace_context = trace_context_obj
 
             logger.debug(f"Successfully retrieved and parsed job {validated_job.id}")
             return validated_job
@@ -444,6 +465,26 @@ class JobStore:
         await self.redis.hset(job_key, "status", status.value.encode("utf-8"))
         logger.debug(f"Updated status of job {job_id} to {status.value}.")
 
+    async def update_job_next_scheduled_run_time(
+        self, job_id: str, run_time: datetime
+    ) -> None:
+        """Updates only the next scheduled run time field for a job.
+
+        This is primarily used to keep job metadata accurate when re-queuing jobs
+        for retries or deferrals via atomic operations.
+        """
+        job_key = f"{JOB_KEY_PREFIX}{job_id}"
+        dt = run_time
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        elif dt.tzinfo != timezone.utc:
+            dt = dt.astimezone(timezone.utc)
+        await self.redis.hset(
+            job_key,
+            "next_scheduled_run_time",
+            dt.isoformat().encode("utf-8"),
+        )
+
     async def increment_job_retries(self, job_id: str) -> int:
         """Atomically increments the 'current_retries' field for a job.
 
@@ -840,7 +881,11 @@ class JobStore:
     async def get_last_process_time(self, unique_key: str) -> Optional[datetime]:
         key = f"last_process:{unique_key}"
         timestamp = await self.redis.get(key)
-        return datetime.fromtimestamp(float(timestamp), timezone.utc) if timestamp else None
+        return (
+            datetime.fromtimestamp(float(timestamp), timezone.utc)
+            if timestamp
+            else None
+        )
 
     async def set_last_process_time(self, unique_key: str, timestamp: datetime) -> None:
         key = f"last_process:{unique_key}"
@@ -864,8 +909,6 @@ class JobStore:
 
     async def batch_get_queue_sizes(self, queue_names: list[str]) -> dict[str, int]:
         """Efficiently get sizes for multiple queues using pipeline"""
-        from .constants import QUEUE_KEY_PREFIX
-
         if not queue_names:
             return {}
 
@@ -873,7 +916,7 @@ class JobStore:
         # No atomicity needed as we're only reading, this improves performance
         async with self.redis.pipeline(transaction=False) as pipe:
             for queue_name in queue_names:
-                queue_key = f"{QUEUE_KEY_PREFIX}{queue_name}"
+                queue_key = self._format_queue_key(queue_name)
                 pipe.zcard(queue_key)
 
             sizes = await pipe.execute()

rrq/telemetry.py

@@ -0,0 +1,129 @@
+"""Pluggable telemetry for RRQ.
+
+RRQ intentionally keeps telemetry optional: the core queue semantics must work
+even when tracing/metrics libraries are missing or misconfigured.
+
+Telemetry is configured per-process via :func:`configure` and used internally by
+RRQClient and RRQWorker.
+"""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager
+from typing import Any, Optional
+
+from .job import Job
+
+
+class EnqueueSpan(AbstractContextManager[Optional[dict[str, str]]]):
+    """Context manager for an enqueue span.
+
+    Entering yields an optional propagation carrier dict to store on the Job.
+    """
+
+    def __enter__(self) -> Optional[dict[str, str]]:
+        return None
+
+    def __exit__(self, exc_type, exc, tb) -> bool:  # type: ignore[override]
+        return False
+
+
+class JobSpan(AbstractContextManager["JobSpan"]):
+    """Context manager for a job execution span."""
+
+    def __enter__(self) -> "JobSpan":
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:  # type: ignore[override]
+        self.close()
+        return False
+
+    def success(self, *, duration_seconds: float) -> None:
+        pass
+
+    def retry(
+        self,
+        *,
+        duration_seconds: float,
+        delay_seconds: Optional[float] = None,
+        reason: Optional[str] = None,
+    ) -> None:
+        pass
+
+    def dlq(
+        self,
+        *,
+        duration_seconds: float,
+        reason: Optional[str] = None,
+        error: Optional[BaseException] = None,
+    ) -> None:
+        pass
+
+    def timeout(
+        self,
+        *,
+        duration_seconds: float,
+        timeout_seconds: float,
+        error_message: Optional[str] = None,
+    ) -> None:
+        pass
+
+    def cancelled(
+        self, *, duration_seconds: float, reason: Optional[str] = None
+    ) -> None:
+        pass
+
+    def close(self) -> None:
+        pass
+
+
+class Telemetry:
+    """Base telemetry implementation (no-op by default)."""
+
+    enabled: bool = False
+
+    def enqueue_span(
+        self, *, job_id: str, function_name: str, queue_name: str
+    ) -> EnqueueSpan:
+        return _NOOP_ENQUEUE_SPAN
+
+    def job_span(
+        self,
+        *,
+        job: Job,
+        worker_id: str,
+        queue_name: str,
+        attempt: int,
+        timeout_seconds: float,
+    ) -> JobSpan:
+        return _NOOP_JOB_SPAN
+
+    def worker_started(self, *, worker_id: str, queues: list[str]) -> None:
+        pass
+
+    def worker_stopped(self, *, worker_id: str) -> None:
+        pass
+
+    def worker_heartbeat(self, *, worker_id: str, health_data: dict[str, Any]) -> None:
+        pass
+
+
+_NOOP_ENQUEUE_SPAN = EnqueueSpan()
+_NOOP_JOB_SPAN = JobSpan()
+_telemetry: Telemetry = Telemetry()
+
+
+def configure(telemetry: Telemetry) -> None:
+    """Configure a process-global telemetry backend."""
+    global _telemetry
+    _telemetry = telemetry
+
+
+def disable() -> None:
+    """Disable RRQ telemetry for the current process."""
+    configure(Telemetry())
+
+
+def get_telemetry() -> Telemetry:
+    """Return the configured telemetry backend (defaults to no-op)."""
+    return _telemetry