rrq 0.2.5__py3-none-any.whl → 0.3.5__py3-none-any.whl

rrq/{rrq.py → cli.py}

@@ -20,10 +20,11 @@ from .worker import RRQWorker
 
 logger = logging.getLogger(__name__)
 
+
 # Helper to load settings for commands
 def _load_app_settings(settings_object_path: str | None = None) -> RRQSettings:
     """Load the settings object from the given path.
-    If not provided, the RRQ_SETTINGS environment variable will be used. 
+    If not provided, the RRQ_SETTINGS environment variable will be used.
     If the environment variable is not set, will create a default settings object.
     RRQ Setting objects, automatically pick up ENVIRONMENT variables starting with RRQ_.
 
@@ -53,10 +54,22 @@ def _load_app_settings(settings_object_path: str | None = None) -> RRQSettings:
 
         return settings_object
     except ImportError:
-        click.echo(click.style(f"ERROR: Could not import settings object '{settings_object_path}'. Make sure it is in PYTHONPATH.", fg="red"), err=True)
+        click.echo(
+            click.style(
+                f"ERROR: Could not import settings object '{settings_object_path}'. Make sure it is in PYTHONPATH.",
+                fg="red",
+            ),
+            err=True,
+        )
         sys.exit(1)
     except Exception as e:
-        click.echo(click.style(f"ERROR: Unexpected error processing settings object '{settings_object_path}': {e}", fg="red"), err=True)
+        click.echo(
+            click.style(
+                f"ERROR: Unexpected error processing settings object '{settings_object_path}': {e}",
+                fg="red",
+            ),
+            err=True,
+        )
         sys.exit(1)
 
 
@@ -73,13 +86,25 @@ async def check_health_async_impl(settings_object_path: str | None = None) -> bo
         logger.debug(f"Successfully connected to Redis: {rrq_settings.redis_dsn}")
 
         health_key_pattern = f"{HEALTH_KEY_PREFIX}*"
-        worker_keys = [key_bytes.decode("utf-8") async for key_bytes in job_store.redis.scan_iter(match=health_key_pattern)]
+        worker_keys = [
+            key_bytes.decode("utf-8")
+            async for key_bytes in job_store.redis.scan_iter(match=health_key_pattern)
+        ]
 
         if not worker_keys:
-            click.echo(click.style("Worker Health Check: FAIL (No active workers found)", fg="red"))
+            click.echo(
+                click.style(
+                    "Worker Health Check: FAIL (No active workers found)", fg="red"
+                )
+            )
             return False
 
-        click.echo(click.style(f"Worker Health Check: Found {len(worker_keys)} active worker(s):", fg="green"))
+        click.echo(
+            click.style(
+                f"Worker Health Check: Found {len(worker_keys)} active worker(s):",
+                fg="green",
+            )
+        )
         for key in worker_keys:
             worker_id = key.split(HEALTH_KEY_PREFIX)[1]
             health_data, ttl = await job_store.get_worker_health(worker_id)
@@ -95,28 +120,49 @@ async def check_health_async_impl(settings_object_path: str | None = None) -> bo
                     f"    TTL: {ttl if ttl is not None else 'N/A'} seconds"
                 )
             else:
-                click.echo(f"  - Worker ID: {click.style(worker_id, bold=True)} - Health data missing/invalid. TTL: {ttl if ttl is not None else 'N/A'}s")
+                click.echo(
+                    f"  - Worker ID: {click.style(worker_id, bold=True)} - Health data missing/invalid. TTL: {ttl if ttl is not None else 'N/A'}s"
+                )
         return True
     except redis.exceptions.ConnectionError as e:
         logger.error(f"Redis connection failed during health check: {e}", exc_info=True)
-        click.echo(click.style(f"Worker Health Check: FAIL - Redis connection error: {e}", fg="red"))
+        click.echo(
+            click.style(
+                f"Worker Health Check: FAIL - Redis connection error: {e}", fg="red"
+            )
+        )
         return False
     except Exception as e:
-        logger.error(f"An unexpected error occurred during health check: {e}", exc_info=True)
-        click.echo(click.style(f"Worker Health Check: FAIL - Unexpected error: {e}", fg="red"))
+        logger.error(
+            f"An unexpected error occurred during health check: {e}", exc_info=True
+        )
+        click.echo(
+            click.style(f"Worker Health Check: FAIL - Unexpected error: {e}", fg="red")
+        )
         return False
     finally:
         if job_store:
             await job_store.aclose()
 
+
 # --- Process Management ---
-def start_rrq_worker_subprocess(is_detached: bool = False, settings_object_path: str | None = None) -> subprocess.Popen | None:
-    """Start an RRQ worker process."""
+def start_rrq_worker_subprocess(
+    is_detached: bool = False,
+    settings_object_path: str | None = None,
+    queues: list[str] | None = None,
+) -> subprocess.Popen | None:
+    """Start an RRQ worker process, optionally for specific queues."""
     command = ["rrq", "worker", "run"]
     if settings_object_path:
         command.extend(["--settings", settings_object_path])
     else:
-        raise ValueError("start_rrq_worker_subprocess called without settings_object_path!")
+        raise ValueError(
+            "start_rrq_worker_subprocess called without settings_object_path!"
+        )
+    # Add queue filters if specified
+    if queues:
+        for q in queues:
+            command.extend(["--queue", q])
 
     logger.info(f"Starting worker subprocess with command: {' '.join(command)}")
     if is_detached:
@@ -139,35 +185,55 @@ def start_rrq_worker_subprocess(is_detached: bool = False, settings_object_path:
     return process
 
 
-def terminate_worker_process(process: subprocess.Popen | None, logger: logging.Logger) -> None:
+def terminate_worker_process(
+    process: subprocess.Popen | None, logger: logging.Logger
+) -> None:
     if not process or process.pid is None:
         logger.debug("No active worker process to terminate.")
         return
 
     try:
         if process.poll() is not None:
-            logger.debug(f"Worker process {process.pid} already terminated (poll returned exit code: {process.returncode}).")
+            logger.debug(
+                f"Worker process {process.pid} already terminated (poll returned exit code: {process.returncode})."
+            )
             return
 
         pgid = os.getpgid(process.pid)
-        logger.info(f"Terminating worker process group for PID {process.pid} (PGID {pgid})...")
+        logger.info(
+            f"Terminating worker process group for PID {process.pid} (PGID {pgid})..."
+        )
         os.killpg(pgid, signal.SIGTERM)
         process.wait(timeout=5)
     except subprocess.TimeoutExpired:
-        logger.warning(f"Worker process {process.pid} did not terminate gracefully (SIGTERM timeout), sending SIGKILL.")
+        logger.warning(
+            f"Worker process {process.pid} did not terminate gracefully (SIGTERM timeout), sending SIGKILL."
+        )
         with suppress(ProcessLookupError):
             os.killpg(os.getpgid(process.pid), signal.SIGKILL)
     except Exception as e:
         logger.error(f"Unexpected error checking worker process {process.pid}: {e}")
 
 
-async def watch_rrq_worker_impl(watch_path: str, settings_object_path: str | None = None) -> None:
+async def watch_rrq_worker_impl(
+    watch_path: str,
+    settings_object_path: str | None = None,
+    queues: list[str] | None = None,
+) -> None:
     if not settings_object_path:
-        click.echo(click.style("ERROR: 'rrq worker watch' requires --settings to be specified.", fg="red"), err=True)
+        click.echo(
+            click.style(
+                "ERROR: 'rrq worker watch' requires --settings to be specified.",
+                fg="red",
+            ),
+            err=True,
+        )
         sys.exit(1)
 
     abs_watch_path = os.path.abspath(watch_path)
-    click.echo(f"Watching for file changes in {abs_watch_path} to restart RRQ worker (app settings: {settings_object_path})...")
+    click.echo(
+        f"Watching for file changes in {abs_watch_path} to restart RRQ worker (app settings: {settings_object_path})..."
+    )
     worker_process: subprocess.Popen | None = None
     loop = asyncio.get_event_loop()
     shutdown_event = asyncio.Event()
@@ -184,20 +250,28 @@ async def watch_rrq_worker_impl(watch_path: str, settings_object_path: str | Non
     signal.signal(signal.SIGTERM, sig_handler)
 
     try:
-        worker_process = start_rrq_worker_subprocess(is_detached=False, settings_object_path=settings_object_path)
+        worker_process = start_rrq_worker_subprocess(
+            is_detached=False,
+            settings_object_path=settings_object_path,
+            queues=queues,
+        )
         async for changes in awatch(abs_watch_path, stop_event=shutdown_event):
-            if shutdown_event.is_set(): 
+            if shutdown_event.is_set():
                 break
-            if not changes: 
+            if not changes:
                 continue
 
             logger.info(f"File changes detected: {changes}. Restarting RRQ worker...")
             if worker_process is not None:
                 terminate_worker_process(worker_process, logger)
             await asyncio.sleep(1)
-            if shutdown_event.is_set(): 
+            if shutdown_event.is_set():
                 break
-            worker_process = start_rrq_worker_subprocess(is_detached=False, settings_object_path=settings_object_path)
+            worker_process = start_rrq_worker_subprocess(
+                is_detached=False,
+                settings_object_path=settings_object_path,
+                queues=queues,
+            )
     except Exception as e:
         logger.error(f"Error in watch_rrq_worker: {e}", exc_info=True)
     finally:
@@ -213,7 +287,8 @@ async def watch_rrq_worker_impl(watch_path: str, settings_object_path: str | Non
 
 # --- Click CLI Definitions ---
 
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+
 
 @click.group(context_settings=CONTEXT_SETTINGS)
 def rrq():
@@ -226,7 +301,6 @@ def rrq():
     pass
 
 
-
 @rrq.group("worker")
 def worker_cli():
     """Manage RRQ workers (run, watch)."""
@@ -234,41 +308,66 @@ def worker_cli():
 
 
 @worker_cli.command("run")
-@click.option("--burst", is_flag=True, help="Run worker in burst mode (process one job/batch then exit). Not Implemented yet.")
-@click.option("--detach", is_flag=True, help="Run the worker in the background (detached).")
 @click.option(
-    "--settings", 
-    "settings_object_path", 
-    type=str, 
-    required=False, 
-    default=None, 
-    help="Python settings path for application worker settings (e.g., myapp.worker_config.rrq_settings)."
+    "--burst",
+    is_flag=True,
+    help="Run worker in burst mode (process one job/batch then exit).",
 )
-def worker_run_command(burst: bool, detach: bool, settings_object_path: str):
+@click.option(
+    "--queue",
+    "queues",
+    type=str,
+    multiple=True,
+    help="Queue(s) to poll. Defaults to settings.default_queue_name.",
+)
+@click.option(
+    "--settings",
+    "settings_object_path",
+    type=str,
+    required=False,
+    default=None,
+    help=(
+        "Python settings path for application worker settings "
+        "(e.g., myapp.worker_config.rrq_settings). "
+        "The specified settings object must include a `job_registry: JobRegistry`."
+    ),
+)
+def worker_run_command(
+    burst: bool,
+    queues: tuple[str, ...],
+    settings_object_path: str,
+):
     """Run an RRQ worker process. Requires --settings."""
     rrq_settings = _load_app_settings(settings_object_path)
 
-    if detach:
-        logger.info("Attempting to start worker in detached (background) mode...")
-        process = start_rrq_worker_subprocess(is_detached=True, settings_object_path=settings_object_path)
-        click.echo(f"Worker initiated in background (PID: {process.pid}). Check logs for status.")
-        return
-
-    if burst:
-        raise NotImplementedError("Burst mode is not implemented yet.")
+    # Determine queues to poll
+    queues_arg = list(queues) if queues else None
+    # Run worker in foreground (burst or continuous mode)
 
-    logger.info(f"Starting RRQ Worker (Burst: {burst}, App Settings: {settings_object_path})")
+    logger.info(
+        f"Starting RRQ Worker (Burst: {burst}, App Settings: {settings_object_path})"
+    )
 
     if not rrq_settings.job_registry:
-        click.echo(click.style("ERROR: No 'job_registry_app'. You must provide a JobRegistry instance in settings.", fg="red"), err=True)
+        click.echo(
+            click.style(
+                "ERROR: No 'job_registry_app'. You must provide a JobRegistry instance in settings.",
+                fg="red",
+            ),
+            err=True,
+        )
         sys.exit(1)
 
-    logger.debug(f"Registered handlers (from effective registry): {rrq_settings.job_registry.get_registered_functions()}")
+    logger.debug(
+        f"Registered handlers (from effective registry): {rrq_settings.job_registry.get_registered_functions()}"
+    )
     logger.debug(f"Effective RRQ settings for worker: {rrq_settings}")
 
     worker_instance = RRQWorker(
         settings=rrq_settings,
         job_registry=rrq_settings.job_registry,
+        queues=queues_arg,
+        burst=burst,
     )
 
     loop = asyncio.get_event_loop()
@@ -296,33 +395,126 @@ def worker_run_command(burst: bool, detach: bool, settings_object_path: str):
     show_default=True,
 )
 @click.option(
-    "--settings", 
-    "settings_object_path", 
-    type=str, 
-    required=False, 
-    default=None, 
-    help="Python settings path for application worker settings (e.g., myapp.worker_config.rrq_settings)."
+    "--settings",
+    "settings_object_path",
+    type=str,
+    required=False,
+    default=None,
+    help=(
+        "Python settings path for application worker settings "
+        "(e.g., myapp.worker_config.rrq_settings). "
+        "The specified settings object must define a `job_registry: JobRegistry`."
+    ),
+)
+@click.option(
+    "--queue",
+    "queues",
+    type=str,
+    multiple=True,
+    help="Queue(s) to poll when restarting worker. Defaults to settings.default_queue_name.",
 )
-def worker_watch_command(path: str, settings_object_path: str):
+def worker_watch_command(
+    path: str,
+    settings_object_path: str,
+    queues: tuple[str, ...],
+):
     """Run the RRQ worker with auto-restart on file changes in PATH. Requires --settings."""
-    asyncio.run(watch_rrq_worker_impl(path, settings_object_path=settings_object_path))
+    # Run watch with optional queue filters
+    asyncio.run(
+        watch_rrq_worker_impl(
+            path,
+            settings_object_path=settings_object_path,
+            queues=list(queues) if queues else None,
+        )
+    )
+
+
+# --- DLQ Requeue CLI Command (delegates to JobStore) ---
 
 
 @rrq.command("check")
 @click.option(
-    "--settings", 
-    "settings_object_path", 
-    type=str, 
-    required=False, 
-    default=None, 
-    help="Python settings path for application worker settings (e.g., myapp.worker_config.rrq_settings)."
+    "--settings",
+    "settings_object_path",
+    type=str,
+    required=False,
+    default=None,
+    help=(
+        "Python settings path for application worker settings "
+        "(e.g., myapp.worker_config.rrq_settings). "
+        "Must include `job_registry: JobRegistry` to identify workers."
+    ),
 )
 def check_command(settings_object_path: str):
     """Perform a health check on active RRQ worker(s). Requires --settings."""
     click.echo("Performing RRQ health check...")
-    healthy = asyncio.run(check_health_async_impl(settings_object_path=settings_object_path))
+    healthy = asyncio.run(
+        check_health_async_impl(settings_object_path=settings_object_path)
+    )
     if healthy:
         click.echo(click.style("Health check PASSED.", fg="green"))
     else:
         click.echo(click.style("Health check FAILED.", fg="red"))
         sys.exit(1)
+
+
+@rrq.group("dlq")
+def dlq_cli():
+    """Manage the Dead Letter Queue (DLQ)."""
+    pass
+
+
+@dlq_cli.command("requeue")
+@click.option(
+    "--settings",
+    "settings_object_path",
+    type=str,
+    required=False,
+    default=None,
+    help=(
+        "Python settings path for application worker settings "
+        "(e.g., myapp.worker_config.rrq_settings). "
+        "Must include `job_registry: JobRegistry` if requeueing requires handler resolution."
+    ),
+)
+@click.option(
+    "--dlq-name",
+    "dlq_name",
+    type=str,
+    required=False,
+    default=None,
+    help="Name of the DLQ (without prefix). Defaults to settings.default_dlq_name.",
+)
+@click.option(
+    "--queue",
+    "target_queue",
+    type=str,
+    required=False,
+    default=None,
+    help="Name of the target queue (without prefix). Defaults to settings.default_queue_name.",
+)
+@click.option(
+    "--limit",
+    type=int,
+    required=False,
+    default=None,
+    help="Maximum number of DLQ jobs to requeue; all if not set.",
+)
+def dlq_requeue_command(
+    settings_object_path: str,
+    dlq_name: str,
+    target_queue: str,
+    limit: int,
+):
+    """Requeue jobs from the dead letter queue back into a live queue."""
+    rrq_settings = _load_app_settings(settings_object_path)
+    dlq_to_use = dlq_name or rrq_settings.default_dlq_name
+    queue_to_use = target_queue or rrq_settings.default_queue_name
+    job_store = JobStore(settings=rrq_settings)
+    click.echo(
+        f"Requeuing jobs from DLQ '{dlq_to_use}' to queue '{queue_to_use}' (limit: {limit or 'all'})..."
+    )
+    count = asyncio.run(job_store.requeue_dlq(dlq_to_use, queue_to_use, limit))
+    click.echo(
+        f"Requeued {count} job(s) from DLQ '{dlq_to_use}' to queue '{queue_to_use}'."
+    )

rrq/client.py

@@ -77,26 +77,44 @@ class RRQClient:
             The created Job object if successfully enqueued, or None if enqueueing was denied
             (e.g., due to a unique key conflict).
         """
-        # print(
-        #     f"DEBUG RRQClient.enqueue: function_name='{function_name}', args={args}, kwargs={kwargs}"
-        # )  # DEBUG
-
+        # Determine job ID and enqueue timestamp
         job_id_to_use = _job_id or str(uuid.uuid4())
+        enqueue_time_utc = datetime.now(UTC)
 
+        # Compute unique lock TTL: cover deferral window if any
+        lock_ttl_seconds = self.settings.default_unique_job_lock_ttl_seconds
+        if _defer_by is not None:
+            # Defer relative to now
+            defer_secs = max(0, int(_defer_by.total_seconds()))
+            lock_ttl_seconds = max(lock_ttl_seconds, defer_secs + 1)
+        elif _defer_until is not None:
+            # Defer until specific datetime
+            dt = _defer_until
+            # Normalize to UTC
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=UTC)
+            elif dt.tzinfo != UTC:
+                dt = dt.astimezone(UTC)
+            diff = (dt - enqueue_time_utc).total_seconds()
+            if diff > 0:
+                lock_ttl_seconds = max(lock_ttl_seconds, int(diff) + 1)
+
+        unique_acquired = False
+        # Acquire unique lock if requested, with TTL covering defer window
         if _unique_key:
             lock_acquired = await self.job_store.acquire_unique_job_lock(
                 unique_key=_unique_key,
-                job_id=job_id_to_use,  # Store current job_id in lock for traceability
-                lock_ttl_seconds=self.settings.default_unique_job_lock_ttl_seconds,
+                job_id=job_id_to_use,
+                lock_ttl_seconds=lock_ttl_seconds,
             )
             if not lock_acquired:
                 logger.info(
                     f"Job with unique key '{_unique_key}' already active or recently run. Enqueue denied."
                 )
                 return None
+            unique_acquired = True
 
         queue_name_to_use = _queue_name or self.settings.default_queue_name
-        enqueue_time_utc = datetime.now(UTC)
 
         # Create the Job instance with all provided details and defaults
         job = Job(
@@ -126,9 +144,6 @@ class RRQClient:
             queue_name=queue_name_to_use,  # Store the target queue name
         )
 
-        # Save the full job definition
-        await self.job_store.save_job_definition(job)
-
         # Determine the score for the sorted set (queue)
         # Score is a millisecond timestamp for when the job should be processed.
         score_dt = enqueue_time_utc  # Default to immediate processing
@@ -145,13 +160,21 @@ class RRQClient:
             score_dt = score_dt.astimezone(UTC)
 
         score_timestamp_ms = int(score_dt.timestamp() * 1000)
-
-        # Add the job ID to the processing queue
-        await self.job_store.add_job_to_queue(
-            queue_name_to_use,
-            job.id,
-            float(score_timestamp_ms),  # Redis ZADD score must be float
-        )
+        # Record when the job is next scheduled to run (for deferred execution)
+        job.next_scheduled_run_time = score_dt
+
+        # Save the full job definition and add to queue (ensure unique lock is released on error)
+        try:
+            await self.job_store.save_job_definition(job)
+            await self.job_store.add_job_to_queue(
+                queue_name_to_use,
+                job.id,
+                float(score_timestamp_ms),
+            )
+        except Exception:
+            if unique_acquired:
+                await self.job_store.release_unique_job_lock(_unique_key)
+            raise
 
         logger.debug(
             f"Enqueued job {job.id} ('{job.function_name}') to queue '{queue_name_to_use}' with score {score_timestamp_ms}"

rrq/constants.py

@@ -15,6 +15,7 @@ DEFAULT_DLQ_NAME: str = "rrq:dlq:default"
 # Redis key prefixes
 JOB_KEY_PREFIX: str = "rrq:job:"
 QUEUE_KEY_PREFIX: str = "rrq:queue:"  # For ZSETs holding job IDs
+DLQ_KEY_PREFIX: str = "rrq:dlq:"  # For lists holding Dead Letter Queue job IDs
 ACTIVE_JOBS_PREFIX: str = (
     "rrq:active:"  # For lists of active jobs per worker (optional, for recovery)
 )

rrq/job.py

@@ -53,8 +53,6 @@ class Job(BaseModel):
         default_factory=lambda: datetime.now(UTC),
         description="Timestamp (UTC) when the job was initially enqueued.",
     )
-    # score: Optional[float] = None # The score in the ZSET, derived from defer_until/defer_by
-    # Not stored in the job hash directly, but used for queueing.
 
     status: JobStatus = Field(
         default=JobStatus.PENDING, description="Current status of the job."
@@ -107,27 +105,3 @@ class Job(BaseModel):
         default=None,
         description="The name of the Dead Letter Queue this job will be moved to if it fails permanently.",
     )
-
-    # For model_config to allow arbitrary types if result is complex and not Pydantic model
-    # class Config:
-    #     arbitrary_types_allowed = True
-
-    # def to_redis_hash(self) -> dict[str, Any]:
-    #     """Prepares the job model for storage as a Redis hash.
-    #     Pydantic's model_dump is good, but we might want to ensure all values are easily
-    #     storable as strings or simple types for Redis, or handle serialization here.
-    #     For now, model_dump with json_encoders should suffice with a good serializer.
-    #     """
-    #     # Using model_dump ensures that Pydantic models are properly serialized (e.g., datetimes to ISO strings)
-    #     # We will use a JSON serializer in JobStore that handles Pydantic models correctly.
-    #     return self.model_dump(exclude_none=True)
-
-    # @classmethod
-    # def from_redis_hash(cls, data: dict[str, Any]) -> "Job":
-    #     """Reconstructs a Job instance from data retrieved from a Redis hash."""""""""
-    #     # Pydantic will handle parsing basic types. Datetimes are expected to be ISO strings.
-    #     # Handle potential None values for args/kwargs if they were excluded from dump
-    #     # data.setdefault("args", None) # Removed
-    #     # data.setdefault("kwargs", None) # Removed
-    #     return cls(**data)
-    pass  # Add pass if class body becomes empty after removing methods, or remove if not needed

rrq/registry.py

@@ -2,8 +2,6 @@
 
 from typing import Any, Callable, Optional
 
-# Potentially: from collections.abc import Callable if more specific async callable needed
-
 
 class JobRegistry:
     """Manages the registration and retrieval of job handler functions.
@@ -74,4 +72,3 @@ class JobRegistry:
 
 
 # Global instance for convenience, though applications might manage their own.
-# job_registry = JobRegistry()

rrq/settings.py

@@ -5,7 +5,6 @@ Settings can be loaded from environment variables (with a prefix of `RRQ_`) or
 from a .env file. Sensible defaults are provided for most settings.
 """
 
-# Import Callable and Awaitable for type hinting hooks
 from typing import Awaitable, Callable, Optional
 
 from pydantic import Field
@@ -95,7 +94,8 @@ class RRQSettings(BaseSettings):
         description="Grace period (in seconds) for active job tasks to finish during worker shutdown.",
     )
     job_registry: Optional[JobRegistry] = Field(
-        default=None, description="Job registry instance, typically provided by the application."
+        default=None,
+        description="Job registry instance, typically provided by the application.",
     )
     model_config = SettingsConfigDict(
         env_prefix="RRQ_",
@@ -104,4 +104,3 @@ class RRQSettings(BaseSettings):
         # env_file=".env",
         # env_file_encoding='utf-8'
     )
-

rrq/store.py

@@ -38,6 +38,23 @@ class JobStore:
         self.redis = AsyncRedis.from_url(
             settings.redis_dsn, decode_responses=False
         )  # Work with bytes initially
+        
+    def _format_queue_key(self, queue_name: str) -> str:
+        """Normalize a queue name or key into a Redis key for ZSET queues."""
+
+        # If already a full key, use it directly
+        if queue_name.startswith(QUEUE_KEY_PREFIX):
+            return queue_name
+        return f"{QUEUE_KEY_PREFIX}{queue_name}"
+
+    def _format_dlq_key(self, dlq_name: str) -> str:
+        """Normalize a DLQ name or key into a Redis key for DLQ lists."""
+        from .constants import DLQ_KEY_PREFIX
+
+        # If already a full key, use it directly
+        if dlq_name.startswith(DLQ_KEY_PREFIX):
+            return dlq_name
+        return f"{DLQ_KEY_PREFIX}{dlq_name}"
 
     async def aclose(self):
         """Closes the Redis connection pool associated with this store."""
@@ -83,9 +100,6 @@ class JobStore:
             job: The Job object to save.
         """
         job_key = f"{JOB_KEY_PREFIX}{job.id}"
-        # print(
-        #     f"DEBUG JobStore.save_job_definition (ENTRY): job.id={job.id}, job.job_args={job.job_args}, job.job_kwargs={job.job_kwargs}, type(job.job_args)={type(job.job_args)}"
-        # )
 
         # Dump model excluding fields handled manually
         job_data_dict = job.model_dump(
@@ -111,9 +125,6 @@ class JobStore:
         if "id" not in final_mapping_for_hset:
             final_mapping_for_hset["id"] = job.id
 
-        # print(
-        #     f"!!! RRQ_JOB_STORE_SAVE (PRINT) (JobID:{job.id}) -> Mapping for HSET: { {k: str(v)[:50] + '...' if isinstance(v, str) and len(v) > 50 else v for k, v in final_mapping_for_hset.items()} }"
-        # )
         if final_mapping_for_hset:  # Avoid HSET with empty mapping
             await self.redis.hset(job_key, mapping=final_mapping_for_hset)
             logger.debug(f"Saved job definition for {job.id} to Redis hash {job_key}.")
@@ -153,7 +164,6 @@ class JobStore:
         if job_args_str and job_args_str.lower() != "null":
             try:
                 job_args_list = json.loads(job_args_str)
-                # print(f"DEBUG get_job_def: Parsed job_args_list = {job_args_list}")
             except json.JSONDecodeError:
                 logger.error(
                     f"Failed to JSON decode 'job_args' for job {job_id} from string: '{job_args_str}'",
@@ -163,7 +173,6 @@ class JobStore:
         if job_kwargs_str and job_kwargs_str.lower() != "null":
             try:
                 job_kwargs_dict = json.loads(job_kwargs_str)
-                # print(f"DEBUG get_job_def: Parsed job_kwargs_dict = {job_kwargs_dict}")
             except json.JSONDecodeError:
                 logger.error(
                     f"Failed to JSON decode 'job_kwargs' for job {job_id} from string: '{job_kwargs_str}'",
@@ -196,12 +205,6 @@ class JobStore:
             )
             validated_job.result = result_obj
 
-            # print(
-            #     f"DEBUG get_job_def (POST-MANUAL-ASSIGN): job_id={job_id}, job.job_args='{validated_job.job_args}', type={type(validated_job.job_args)}"
-            # )
-            # print(
-            #     f"!!! RRQ_JOB_STORE_GET (POST_CREATE_VALIDATED) -> JobID:{validated_job.id}, Status:{validated_job.status.value if validated_job.status else None}, Retries:{validated_job.current_retries}"
-            # )
             logger.debug(f"Successfully retrieved and parsed job {validated_job.id}")
             return validated_job
         except Exception as e_val:
@@ -224,7 +227,7 @@ class JobStore:
             job_id: The ID of the job to add.
             score: The score (float) determining the job's position/priority in the queue.
         """
-        queue_key = f"{QUEUE_KEY_PREFIX}{queue_name}"
+        queue_key = self._format_queue_key(queue_name)
         await self.redis.zadd(
             queue_key, {job_id.encode("utf-8"): score}
         )  # Store job_id as bytes
@@ -243,7 +246,7 @@ class JobStore:
         Returns:
             A list of job IDs as strings.
         """
-        queue_key = f"{QUEUE_KEY_PREFIX}{queue_name}"
+        queue_key = self._format_queue_key(queue_name)
         job_ids_bytes = await self.redis.zrange(queue_key, start, end)
         return [job_id.decode("utf-8") for job_id in job_ids_bytes]
 
@@ -259,7 +262,7 @@ class JobStore:
         """
         if count <= 0:
             return []
-        queue_key = f"{QUEUE_KEY_PREFIX}{queue_name}"
+        queue_key = self._format_queue_key(queue_name)
         now_ms = int(datetime.now(UTC).timestamp() * 1000)
         # Fetch jobs with score from -inf up to current time, limit by count
         job_ids_bytes = await self.redis.zrangebyscore(
@@ -348,7 +351,7 @@ class JobStore:
             completion_time: The timestamp when the job failed permanently.
         """
         job_key = f"{JOB_KEY_PREFIX}{job_id}"
-        dlq_redis_key = f"{QUEUE_KEY_PREFIX}{dlq_name}"
+        dlq_redis_key = self._format_dlq_key(dlq_name)
 
         # Ensure complex fields are properly handled if needed (error could be complex)
         # For now, assuming simple string error message
@@ -365,6 +368,42 @@ class JobStore:
             pipe.expire(job_key, DEFAULT_DLQ_RESULT_TTL_SECONDS)
             results = await pipe.execute()
         logger.info(f"Moved job {job_id} to DLQ '{dlq_redis_key}'. Results: {results}")
+    
+    async def requeue_dlq(
+        self,
+        dlq_name: str,
+        target_queue: str,
+        limit: int | None = None,
+    ) -> int:
+        """Requeue jobs from the Dead Letter Queue back into a live queue.
+
+        Pops jobs from the DLQ list and adds them to the target queue with current timestamp.
+
+        Args:
+            dlq_name: Name of the DLQ (without prefix).
+            target_queue: Name of the target queue (without prefix).
+            limit: Maximum number of jobs to requeue; all if None.
+
+        Returns:
+            Number of jobs requeued.
+        """
+        jobs_requeued = 0
+        dlq_key = self._format_dlq_key(dlq_name)
+        # Continue popping until limit is reached or DLQ is empty
+        while limit is None or jobs_requeued < limit:
+            job_id_bytes = await self.redis.rpop(dlq_key)
+            if not job_id_bytes:
+                break
+            job_id = job_id_bytes.decode("utf-8")
+            # Use current time for re-enqueue score
+            now_ms = int(datetime.now(UTC).timestamp() * 1000)
+            await self.add_job_to_queue(
+                self._format_queue_key(target_queue),
+                job_id,
+                now_ms,
+            )
+            jobs_requeued += 1
+        return jobs_requeued
 
     async def get_job_lock_owner(self, job_id: str) -> Optional[str]:
         """Gets the current owner (worker ID) of a job's processing lock, if held.
@@ -389,7 +428,7 @@ class JobStore:
         Returns:
             The number of elements removed (0 or 1).
         """
-        queue_key = f"{QUEUE_KEY_PREFIX}{queue_name}"
+        queue_key = self._format_queue_key(queue_name)
         removed_count = await self.redis.zrem(queue_key, job_id.encode("utf-8"))
         count = int(removed_count)  # Ensure int
         if count > 0:
@@ -469,16 +508,12 @@ class JobStore:
 
         # Serialize result to JSON string
         try:
-            # Use pydantic-core for robust serialization if available, else standard json
+            # Use pydantic JSON serialization if available, else standard JSON dump
             if hasattr(result, "model_dump_json"):
                 result_str = result.model_dump_json()
-            elif isinstance(result, str):
-                result_str = result  # Store plain strings directly if not JSON-like?
-                # Let's stick to JSON encoding everything for consistency in save/load.
-                # If it's already a string, json.dumps adds quotes.
-                result_str = json.dumps(result)
             else:
-                result_str = json.dumps(result, default=str)  # Handle datetimes etc.
+                # Always JSON-encode the result, converting unknown types to strings
+                result_str = json.dumps(result, default=str)
         except TypeError as e:
             logger.error(
                 f"Failed to serialize result for job {job_id}: {e}", exc_info=True

rrq/worker.py

@@ -48,6 +48,7 @@ class RRQWorker:
         job_registry: JobRegistry,
         queues: Optional[list[str]] = None,
         worker_id: Optional[str] = None,
+        burst: bool = False,
     ):
         """Initializes the RRQWorker.
 
@@ -73,6 +74,8 @@ class RRQWorker:
             worker_id
             or f"{DEFAULT_WORKER_ID_PREFIX}{os.getpid()}_{uuid.uuid4().hex[:6]}"
         )
+        # Burst mode: process existing jobs then exit
+        self.burst = burst
 
         self._semaphore = asyncio.Semaphore(self.settings.worker_concurrency)
         self._running_tasks: set[asyncio.Task] = set()
@@ -144,7 +147,14 @@ class RRQWorker:
                             f"Worker {self.worker_id} polling for up to {jobs_to_fetch} jobs..."
                         )
                         self.status = "polling"
-                    await self._poll_for_jobs(jobs_to_fetch)
+                    # Poll for jobs and get count of jobs started
+                    fetched_count = await self._poll_for_jobs(jobs_to_fetch)
+                    # In burst mode, exit when no new jobs and no tasks running
+                    if self.burst and fetched_count == 0 and not self._running_tasks:
+                        logger.info(
+                            f"Worker {self.worker_id} burst mode complete: no more jobs."
+                        )
+                        break
                 else:
                     if self.status != "idle (concurrency limit)":
                         logger.debug(
@@ -224,6 +234,8 @@ class RRQWorker:
                     exc_info=True,
                 )
                 await asyncio.sleep(1)  # Avoid tight loop on polling error
+        # For burst mode, return number of jobs fetched in this poll
+        return fetched_count
 
     async def _try_process_job(self, job_id: str, queue_name: str) -> bool:
         """Attempts to lock, fetch definition, and start the execution task for a specific job.
@@ -805,7 +817,8 @@ class RRQWorker:
         delay_seconds = min(max_delay, base_delay * (2 ** (retry_attempt - 1)))
         delay_ms = int(delay_seconds * 1000)
         logger.debug(
-            f"Calculated backoff for job {job.id} (attempt {retry_attempt}): {delay_ms}ms"
+            f"Calculated backoff for job {job.id} (attempt {retry_attempt}): "
+            f"base_delay={base_delay}s, max_delay={max_delay}s -> {delay_ms}ms"
         )
         return delay_ms
 
@@ -893,5 +906,6 @@ class RRQWorker:
         if self.client:  # Check if client exists before closing
             await self.client.close()
         if self.job_store:
-            await self.job_store.close()
+            # Close the Redis connection pool
+            await self.job_store.aclose()
         logger.info(f"[{self.worker_id}] RRQ worker closed.")

{rrq-0.2.5.dist-info → rrq-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rrq
-Version: 0.2.5
+Version: 0.3.5
 Summary: RRQ is a Python library for creating reliable job queues using Redis and asyncio
 Project-URL: Homepage, https://github.com/getresq/rrq
 Project-URL: Bug Tracker, https://github.com/getresq/rrq/issues
@@ -21,17 +21,12 @@ Requires-Dist: redis[hiredis]<6,>=4.2.0
 Requires-Dist: watchfiles>=0.19.0
 Provides-Extra: dev
 Requires-Dist: pytest-asyncio>=0.26.0; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
 Requires-Dist: pytest>=8.3.5; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # RRQ: Reliable Redis Queue
 
- ____    ____     ___
-|  _ \  |  _ \   / _ \
-| |_) | | |_) | | | | |
-|  _ <  |  _ <  | |_| |
-|_| \_\ |_| \_\  \__\_\
-
 RRQ is a Python library for creating reliable job queues using Redis and `asyncio`, inspired by [ARQ (Async Redis Queue)](https://github.com/samuelcolvin/arq). It focuses on providing at-least-once job processing semantics with features like automatic retries, job timeouts, dead-letter queues, and graceful worker shutdown.
 
 ## Core Components
@@ -58,10 +53,20 @@ RRQ is a Python library for creating reliable job queues using Redis and `asynci
 *   **Worker Health Checks**: Workers periodically update a health key in Redis with a TTL, allowing monitoring systems to track active workers.
 *   **Deferred Execution**: Jobs can be scheduled to run at a future time using `_defer_by` or `_defer_until`.
     *Note: Using deferral with a specific `_job_id` will effectively reschedule the job associated with that ID to the new time, overwriting its previous definition and score. It does not create multiple distinct scheduled jobs with the same ID.*
+    *To batch multiple enqueue calls into a single deferred job (and prevent duplicates within the defer window), combine `_unique_key` with `_defer_by`. For example:*
+
+      ```python
+      await client.enqueue(
+          "process_updates",
+          item_id=123,
+          _unique_key="update:123",
+          _defer_by=10,
+      )
+      ```
 
 ## Basic Usage
 
-*(See [`rrq_example.py`](examples/rrq_example.py) in the project root for a runnable example)*
+*(See [`rrq_example.py`](https://github.com/GetResQ/rrq/tree/master/example) in the project root for a runnable example)*
 
 **1. Define Handlers:**
 
@@ -165,10 +170,9 @@ rrq <command> [options]
 
 - **`worker run`**: Run an RRQ worker process to process jobs from queues.
   ```bash
-  rrq worker run [--burst] [--detach] --settings <settings_path>
+  rrq worker run [--burst] --settings <settings_path>
   ```
   - `--burst`: Run in burst mode (process one job/batch then exit).
-  - `--detach`: Run the worker in the background.
   - `--settings`: Python settings path for application worker settings (e.g., `myapp.worker_config.rrq_settings`).
 
 - **`worker watch`**: Run an RRQ worker with auto-restart on file changes in a specified directory.

rrq-0.3.5.dist-info/RECORD

@@ -0,0 +1,15 @@
+rrq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rrq/cli.py,sha256=B9ra9ipZGRto423uG1vGEP-_jLshfl2XrCTNqixBMiQ,16562
+rrq/client.py,sha256=5_bmZ05LKIfY9WFSKU-nYawEupsnrnHT2HewXfC2Ahg,7831
+rrq/constants.py,sha256=F_uZgBI3h00MctnEjBjiCGMrg5jUaz5Bz9I1vkyqNrs,1654
+rrq/exc.py,sha256=NJq3C7pUfcd47AB8kghIN8vdY0l90UrsHQEg4McBHP8,1281
+rrq/job.py,sha256=eUbl33QDqDMXPKpo-0dl0Mp29LWWmtbBgRw0sclcwJ4,4011
+rrq/registry.py,sha256=E9W_zx3QiKTBwMOGearaNpDKBDB87JIn0RlMQ3sAcP0,2925
+rrq/settings.py,sha256=BPKP4XjG7z475gqRgHZt4-IvvOs8uZefq4fPfD2Bepk,4350
+rrq/store.py,sha256=teO0Af8hzBiu7-dFn6_2lz5X90LAZXmtg0VDZuQoAwk,24972
+rrq/worker.py,sha256=y0UTziZVh4QbOPv24b8cqbm_xDBM0HtJLwPNYsJPWnE,40706
+rrq-0.3.5.dist-info/METADATA,sha256=0sqVBY1QbqT2GJshYbz_nkKPYfStjZVKE3ICDvv7jdU,9224
+rrq-0.3.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+rrq-0.3.5.dist-info/entry_points.txt,sha256=f8eFjk2ygDSyu9USwXGj5IM8xeyQqZgDa1rSrCj4Mis,36
+rrq-0.3.5.dist-info/licenses/LICENSE,sha256=XDvu5hKdS2-_ByiSj3tiu_3zSsrXXoJsgbILGoMpKCw,554
+rrq-0.3.5.dist-info/RECORD,,

{rrq-0.2.5.dist-info → rrq-0.3.5.dist-info}/entry_points.txt

@@ -1,2 +1,2 @@
 [console_scripts]
-rrq = rrq.rrq:rrq
+rrq = rrq.cli:rrq

rrq-0.2.5.dist-info/RECORD

@@ -1,15 +0,0 @@
-rrq/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-rrq/client.py,sha256=2CXHxu-5TnTISqYfHPHNiTUKtdlwzSNWlSALC0tivvA,6778
-rrq/constants.py,sha256=_BY0iIyztTyoaA8HU43JPG0yGtm_Fv05t-zC3B9al4Q,1574
-rrq/exc.py,sha256=NJq3C7pUfcd47AB8kghIN8vdY0l90UrsHQEg4McBHP8,1281
-rrq/job.py,sha256=sjABmhevzen2ufTwSbj86lPYQCW_HZLae1EneCB1ZX8,5571
-rrq/registry.py,sha256=3wqwwIvqbG4zWRAA8UOuL7_AXOWFHYQA9Mxq6Ad63Xo,3048
-rrq/rrq.py,sha256=b3FeT_PU9bN8OfPhC99-i_b1OgIdV6MokyKepi8OFPw,13054
-rrq/settings.py,sha256=o9XdJ85mbCwJYDVKjVMTBaK8VFZWMGdN_Av9T3fIY7M,4397
-rrq/store.py,sha256=6AZsbK4hnNWMueAFtd0UjPqS8TJOPNLhuSufe214VBM,24352
-rrq/worker.py,sha256=3VybAINLiBrnseisuqZt744nFE-6-WD0CBL8mlN437o,39952
-rrq-0.2.5.dist-info/METADATA,sha256=yDdof--bDhs02JP35naVvvOMma-TaEhaPU_fj5eEpyE,8984
-rrq-0.2.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-rrq-0.2.5.dist-info/entry_points.txt,sha256=tpzz5voYGwoIv6ir-UBUTmkCl1HVtGLTW7An80RUCIk,36
-rrq-0.2.5.dist-info/licenses/LICENSE,sha256=XDvu5hKdS2-_ByiSj3tiu_3zSsrXXoJsgbILGoMpKCw,554
-rrq-0.2.5.dist-info/RECORD,,