rrq 0.2.5__py3-none-any.whl

rrq/client.py

@@ -0,0 +1,159 @@
+"""This module defines the RRQClient, used for enqueuing jobs into the RRQ system."""
+
+import logging
+import uuid
+from datetime import UTC, datetime, timedelta
+from typing import Any, Optional
+
+from .job import Job, JobStatus
+from .settings import RRQSettings
+from .store import JobStore
+
+logger = logging.getLogger(__name__)
+
+
+class RRQClient:
+    """Client interface for interacting with the RRQ (Reliable Redis Queue) system.
+
+    Provides methods primarily for enqueuing jobs.
+    """
+
+    def __init__(self, settings: RRQSettings, job_store: Optional[JobStore] = None):
+        """Initializes the RRQClient.
+
+        Args:
+            settings: The RRQSettings instance containing configuration.
+            job_store: Optional JobStore instance. If not provided, a new one
+                       will be created based on the settings. This allows sharing
+                       a JobStore instance across multiple components.
+        """
+        self.settings = settings
+        # If job_store is not provided, create one. This allows for flexibility:
+        # - External management of JobStore (e.g., passed from an application context)
+        # - Client creates its own if used standalone.
+        if job_store:
+            self.job_store = job_store
+            self._created_store_internally = False
+        else:
+            self.job_store = JobStore(settings=self.settings)
+            self._created_store_internally = True
+
+    async def close(self) -> None:
+        """Closes the underlying JobStore's Redis connection if it was created internally by this client."""
+        if self._created_store_internally:
+            await self.job_store.aclose()
+
+    async def enqueue(
+        self,
+        function_name: str,
+        *args: Any,
+        _queue_name: Optional[str] = None,
+        _job_id: Optional[str] = None,
+        _unique_key: Optional[str] = None,
+        _max_retries: Optional[int] = None,
+        _job_timeout_seconds: Optional[int] = None,
+        _defer_until: Optional[datetime] = None,
+        _defer_by: Optional[timedelta] = None,
+        _result_ttl_seconds: Optional[int] = None,
+        **kwargs: Any,
+    ) -> Optional[Job]:
+        """Enqueues a job to be processed by RRQ workers.
+
+        Args:
+            function_name: The registered name of the handler function to execute.
+            *args: Positional arguments to pass to the handler function.
+            _queue_name: Specific queue to enqueue the job to. Defaults to `RRQSettings.default_queue_name`.
+            _job_id: User-provided job ID for idempotency or tracking. If None, a UUID is generated.
+            _unique_key: If provided, ensures that only one job with this key is active or recently completed.
+                         Uses a Redis lock with `default_unique_job_lock_ttl_seconds`.
+            _max_retries: Maximum number of retries for this specific job. Overrides `RRQSettings.default_max_retries`.
+            _job_timeout_seconds: Timeout (in seconds) for this specific job. Overrides `RRQSettings.default_job_timeout_seconds`.
+            _defer_until: A specific datetime (UTC recommended) when the job should become available for processing.
+            _defer_by: A timedelta relative to now, specifying when the job should become available.
+            _result_ttl_seconds: Time-to-live (in seconds) for the result of this specific job. Overrides `RRQSettings.default_result_ttl_seconds`.
+            **kwargs: Keyword arguments to pass to the handler function.
+
+        Returns:
+            The created Job object if successfully enqueued, or None if enqueueing was denied
+            (e.g., due to a unique key conflict).
+        """
+        # print(
+        #     f"DEBUG RRQClient.enqueue: function_name='{function_name}', args={args}, kwargs={kwargs}"
+        # )  # DEBUG
+
+        job_id_to_use = _job_id or str(uuid.uuid4())
+
+        if _unique_key:
+            lock_acquired = await self.job_store.acquire_unique_job_lock(
+                unique_key=_unique_key,
+                job_id=job_id_to_use,  # Store current job_id in lock for traceability
+                lock_ttl_seconds=self.settings.default_unique_job_lock_ttl_seconds,
+            )
+            if not lock_acquired:
+                logger.info(
+                    f"Job with unique key '{_unique_key}' already active or recently run. Enqueue denied."
+                )
+                return None
+
+        queue_name_to_use = _queue_name or self.settings.default_queue_name
+        enqueue_time_utc = datetime.now(UTC)
+
+        # Create the Job instance with all provided details and defaults
+        job = Job(
+            id=job_id_to_use,
+            function_name=function_name,
+            job_args=list(args),
+            job_kwargs=kwargs,
+            enqueue_time=enqueue_time_utc,
+            status=JobStatus.PENDING,
+            current_retries=0,
+            max_retries=(
+                _max_retries
+                if _max_retries is not None
+                else self.settings.default_max_retries
+            ),
+            job_timeout_seconds=(
+                _job_timeout_seconds
+                if _job_timeout_seconds is not None
+                else self.settings.default_job_timeout_seconds
+            ),
+            result_ttl_seconds=(
+                _result_ttl_seconds
+                if _result_ttl_seconds is not None
+                else self.settings.default_result_ttl_seconds
+            ),
+            job_unique_key=_unique_key,
+            queue_name=queue_name_to_use,  # Store the target queue name
+        )
+
+        # Save the full job definition
+        await self.job_store.save_job_definition(job)
+
+        # Determine the score for the sorted set (queue)
+        # Score is a millisecond timestamp for when the job should be processed.
+        score_dt = enqueue_time_utc  # Default to immediate processing
+        if _defer_until:
+            score_dt = _defer_until
+        elif _defer_by:
+            score_dt = enqueue_time_utc + _defer_by
+
+        # Ensure score_dt is timezone-aware (UTC) if it's naive from user input
+        if score_dt.tzinfo is None:
+            score_dt = score_dt.replace(tzinfo=UTC)
+        elif score_dt.tzinfo != UTC:
+            # Convert to UTC if it's aware but not UTC
+            score_dt = score_dt.astimezone(UTC)
+
+        score_timestamp_ms = int(score_dt.timestamp() * 1000)
+
+        # Add the job ID to the processing queue
+        await self.job_store.add_job_to_queue(
+            queue_name_to_use,
+            job.id,
+            float(score_timestamp_ms),  # Redis ZADD score must be float
+        )
+
+        logger.debug(
+            f"Enqueued job {job.id} ('{job.function_name}') to queue '{queue_name_to_use}' with score {score_timestamp_ms}"
+        )
+        return job

rrq/constants.py

@@ -0,0 +1,42 @@
+"""This module defines constants used throughout the RRQ (Reliable Redis Queue) system.
+
+These constants include Redis key prefixes, default queue names, and default
+configuration values for job processing and worker behavior.
+"""
+
+# RRQ Constants
+
+# Default queue name if not specified
+DEFAULT_QUEUE_NAME: str = "rrq:queue:default"
+
+# Default Dead Letter Queue name
+DEFAULT_DLQ_NAME: str = "rrq:dlq:default"
+
+# Redis key prefixes
+JOB_KEY_PREFIX: str = "rrq:job:"
+QUEUE_KEY_PREFIX: str = "rrq:queue:"  # For ZSETs holding job IDs
+ACTIVE_JOBS_PREFIX: str = (
+    "rrq:active:"  # For lists of active jobs per worker (optional, for recovery)
+)
+LOCK_KEY_PREFIX: str = "rrq:lock:job:"  # For job processing locks
+UNIQUE_JOB_LOCK_PREFIX: str = "rrq:lock:unique:"  # For user-defined unique job keys
+HEALTH_KEY_PREFIX: str = "rrq:health:worker:"
+RETRY_COUNTER_PREFIX: str = (
+    "rrq:retry_count:"  # Potentially, if not stored directly in job hash
+)
+
+# Default job settings (can be overridden by RRQSettings or per job)
+DEFAULT_MAX_RETRIES: int = 5
+DEFAULT_JOB_TIMEOUT_SECONDS: int = 300  # 5 minutes
+DEFAULT_LOCK_TIMEOUT_EXTENSION_SECONDS: int = (
+    60  # How much longer lock should live than job_timeout
+)
+DEFAULT_RESULT_TTL_SECONDS: int = 3600 * 24  # 1 day
+DEFAULT_DLQ_RESULT_TTL_SECONDS: int = 3600 * 24 * 7  # 7 days for DLQ job details
+DEFAULT_UNIQUE_JOB_LOCK_TTL_SECONDS: int = 3600 * 6  # 6 hours for unique job lock
+
+# Poll delay for worker
+DEFAULT_POLL_DELAY_SECONDS: float = 0.1
+
+# Default worker ID if not specified
+DEFAULT_WORKER_ID_PREFIX: str = "rrq_worker_"

rrq/exc.py

@@ -0,0 +1,46 @@
+"""This module defines custom exceptions for the RRQ (Reliable Redis Queue) system."""
+
+from typing import Optional
+
+
+class RRQError(Exception):
+    """Base class for all RRQ specific errors."""
+
+    pass
+
+
+class RetryJob(RRQError):
+    """Exception raised by a job handler to signal that the job should be retried.
+
+    This allows a handler to explicitly request a retry, potentially with a custom delay,
+    rather than relying on automatic retries for general exceptions.
+    """
+
+    def __init__(
+        self,
+        message: str = "Job requested retry",
+        defer_seconds: Optional[float] = None,
+    ):
+        """
+        Args:
+            message: Optional message describing why the retry is requested.
+            defer_seconds: Optional custom delay in seconds before the job is re-queued.
+                         If None, the worker will use its default backoff strategy.
+        """
+        super().__init__(message)
+        self.defer_seconds = defer_seconds
+
+
+class JobNotFound(RRQError):
+    """Exception raised when a job definition cannot be found in the JobStore."""
+
+    pass
+
+
+class MaxRetriesExceeded(Exception):
+    """Raised when a job fails after reaching its maximum retry limit."""
+
+    pass
+
+
+# Add other RRQ-specific exceptions here as needed

rrq/job.py

@@ -0,0 +1,133 @@
+"""This module defines the core data structures for jobs in the RRQ system,
+including the Job model and JobStatus enumeration.
+"""
+
+import uuid
+from datetime import UTC, datetime
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class JobStatus(str, Enum):
+    """Represents the lifecycle status of a job within the RRQ system."""
+
+    PENDING = "PENDING"  # Job enqueued, awaiting processing by a worker.
+    ACTIVE = "ACTIVE"  # Job picked up by a worker and is currently being processed.
+    COMPLETED = "COMPLETED"  # Job processed successfully.
+    FAILED = (
+        "FAILED"  # Job failed after all retry attempts or was a non-retryable failure.
+    )
+    RETRYING = "RETRYING"  # Job failed, an attempt will be made to re-process it after a delay.
+    # NOT_FOUND might be a status for queries, but not stored on the job itself typically
+
+
+def new_job_id() -> str:
+    """Generates a new unique job ID (UUID4)."""
+    return str(uuid.uuid4())
+
+
+class Job(BaseModel):
+    """Represents a job to be processed by an RRQ worker.
+
+    This model encapsulates all the information related to a job, including its
+    identity, execution parameters, status, and results.
+    """
+
+    id: str = Field(
+        default_factory=new_job_id, description="Unique identifier for the job."
+    )
+    function_name: str = Field(
+        description="Name of the handler function to execute for this job."
+    )
+    job_args: list[Any] = Field(
+        default_factory=list,
+        description="Positional arguments for the handler function.",
+    )
+    job_kwargs: dict[str, Any] = Field(
+        default_factory=dict, description="Keyword arguments for the handler function."
+    )
+
+    enqueue_time: datetime = Field(
+        default_factory=lambda: datetime.now(UTC),
+        description="Timestamp (UTC) when the job was initially enqueued.",
+    )
+    # score: Optional[float] = None # The score in the ZSET, derived from defer_until/defer_by
+    # Not stored in the job hash directly, but used for queueing.
+
+    status: JobStatus = Field(
+        default=JobStatus.PENDING, description="Current status of the job."
+    )
+    current_retries: int = Field(
+        default=0, description="Number of retry attempts made so far."
+    )
+    next_scheduled_run_time: Optional[datetime] = Field(
+        default=None,
+        description="Timestamp (UTC) when the job is next scheduled to run (for retries/deferrals).",
+    )
+
+    # Execution control parameters, can be overridden from worker defaults.
+    max_retries: int = Field(
+        default=3, description="Maximum number of retry attempts allowed for this job."
+    )
+    job_timeout_seconds: Optional[int] = Field(
+        default=None,
+        description="Optional per-job execution timeout in seconds. Overrides worker default if set.",
+    )
+    result_ttl_seconds: Optional[int] = Field(
+        default=None,
+        description="Optional Time-To-Live (in seconds) for the job's result. Overrides worker default if set.",
+    )
+
+    # Optional key for ensuring job uniqueness if provided during enqueue.
+    job_unique_key: Optional[str] = Field(
+        default=None, description="Optional key for ensuring job uniqueness."
+    )
+
+    # Fields populated upon job completion or failure.
+    completion_time: Optional[datetime] = Field(
+        default=None,
+        description="Timestamp (UTC) when the job finished (completed or failed permanently).",
+    )
+    result: Optional[Any] = Field(
+        default=None,
+        description="Result of the job if successful, or error details if failed.",
+    )
+    last_error: Optional[str] = Field(
+        default=None,
+        description="String representation of the last error encountered during processing.",
+    )
+
+    # Optional routing hints (currently informational, could be used for advanced routing).
+    queue_name: Optional[str] = Field(
+        default=None, description="The name of the queue this job was last enqueued on."
+    )
+    dlq_name: Optional[str] = Field(
+        default=None,
+        description="The name of the Dead Letter Queue this job will be moved to if it fails permanently.",
+    )
+
+    # For model_config to allow arbitrary types if result is complex and not Pydantic model
+    # class Config:
+    #     arbitrary_types_allowed = True
+
+    # def to_redis_hash(self) -> dict[str, Any]:
+    #     """Prepares the job model for storage as a Redis hash.
+    #     Pydantic's model_dump is good, but we might want to ensure all values are easily
+    #     storable as strings or simple types for Redis, or handle serialization here.
+    #     For now, model_dump with json_encoders should suffice with a good serializer.
+    #     """
+    #     # Using model_dump ensures that Pydantic models are properly serialized (e.g., datetimes to ISO strings)
+    #     # We will use a JSON serializer in JobStore that handles Pydantic models correctly.
+    #     return self.model_dump(exclude_none=True)
+
+    # @classmethod
+    # def from_redis_hash(cls, data: dict[str, Any]) -> "Job":
+    #     """Reconstructs a Job instance from data retrieved from a Redis hash."""""""""
+    #     # Pydantic will handle parsing basic types. Datetimes are expected to be ISO strings.
+    #     # Handle potential None values for args/kwargs if they were excluded from dump
+    #     # data.setdefault("args", None) # Removed
+    #     # data.setdefault("kwargs", None) # Removed
+    #     return cls(**data)
+    pass  # Add pass if class body becomes empty after removing methods, or remove if not needed

rrq/registry.py

@@ -0,0 +1,77 @@
+"""This module provides the JobRegistry class for managing and retrieving job handler functions."""
+
+from typing import Any, Callable, Optional
+
+# Potentially: from collections.abc import Callable if more specific async callable needed
+
+
+class JobRegistry:
+    """Manages the registration and retrieval of job handler functions.
+
+    Handlers are asynchronous functions that perform the actual work of a job.
+    They are registered with a unique name, which is used by the RRQClient to
+    enqueue jobs and by the RRQWorker to look up the appropriate handler for execution.
+    """
+
+    def __init__(self) -> None:
+        """Initializes an empty job registry."""
+        self._handlers: dict[str, Callable[..., Any]] = {}
+
+    def register(
+        self, name: str, handler: Callable[..., Any], replace: bool = False
+    ) -> None:
+        """Registers a job handler function.
+
+        Args:
+            name: The unique name for this handler. Used when enqueuing jobs.
+            handler: The asynchronous callable function that will execute the job.
+                     It should typically accept a context dictionary as its first argument,
+                     followed by job-specific arguments and keyword arguments.
+            replace: If True, an existing handler with the same name will be replaced.
+                     If False (default) and a handler with the same name exists,
+                     a ValueError is raised.
+
+        Raises:
+            ValueError: If a handler with the same name is already registered and `replace` is False,
+                        or if the provided handler is not callable.
+        """
+        if not callable(handler):
+            raise ValueError(f"Handler for '{name}' must be a callable.")
+        if name in self._handlers and not replace:
+            raise ValueError(
+                f"Handler with name '{name}' already registered. Set replace=True to override."
+            )
+        self._handlers[name] = handler
+
+    def unregister(self, name: str) -> None:
+        """Unregisters a job handler function.
+
+        Args:
+            name: The name of the handler to unregister.
+        """
+        # If the handler exists, remove it. Otherwise, do nothing.
+        if name in self._handlers:
+            del self._handlers[name]
+
+    def get_handler(self, name: str) -> Optional[Callable[..., Any]]:
+        """Retrieves a registered job handler function by its name.
+
+        Args:
+            name: The name of the handler to retrieve.
+
+        Returns:
+            The callable handler function if found, otherwise None.
+        """
+        return self._handlers.get(name)
+
+    def get_registered_functions(self) -> list[str]:
+        """Returns a list of names of all registered handler functions."""
+        return list(self._handlers.keys())
+
+    def clear(self) -> None:
+        """Clears all registered handlers from the registry."""
+        self._handlers.clear()
+
+
+# Global instance for convenience, though applications might manage their own.
+# job_registry = JobRegistry()