rrq 0.3.7__py3-none-any.whl → 0.5.0__py3-none-any.whl

rrq/__init__.py

@@ -0,0 +1,14 @@
+from .cron import CronJob, CronSchedule
+from .worker import RRQWorker
+from .client import RRQClient
+from .registry import JobRegistry
+from .settings import RRQSettings
+
+__all__ = [
+    "CronJob",
+    "CronSchedule",
+    "RRQWorker",
+    "RRQClient",
+    "JobRegistry",
+    "RRQSettings",
+]

rrq/cli.py

@@ -7,6 +7,9 @@ import os
 import signal
 import subprocess
 import sys
+import time
+
+# import multiprocessing # No longer needed directly, os.cpu_count() is sufficient
 from contextlib import suppress
 
 import click
@@ -30,6 +33,29 @@ logger = logging.getLogger(__name__)
 
 
 # Helper to load settings for commands
+def _resolve_settings_source(
+    settings_object_path: str | None = None,
+) -> tuple[str | None, str]:
+    """Resolve the settings path and its source.
+
+    Returns:
+        A tuple of (settings_path, source_description)
+    """
+    if settings_object_path is not None:
+        return settings_object_path, "--settings parameter"
+
+    env_setting = os.getenv("RRQ_SETTINGS")
+    if env_setting is not None:
+        # Check if a .env file exists to give more specific info
+        if DOTENV_AVAILABLE and find_dotenv(usecwd=True):
+            # We can't definitively know if it came from .env or system env,
+            # but we can indicate both are possible
+            return env_setting, "RRQ_SETTINGS env var (system or .env)"
+        return env_setting, "RRQ_SETTINGS env var"
+
+    return None, "built-in defaults"
+
+
 def _load_app_settings(settings_object_path: str | None = None) -> RRQSettings:
     """Load the settings object from the given path.
     If not provided, the RRQ_SETTINGS environment variable will be used.
@@ -142,7 +168,12 @@ async def check_health_async_impl(settings_object_path: str | None = None) -> bo
                 )
         return True
     except redis.exceptions.ConnectionError as e:
-        logger.error(f"Redis connection failed during health check: {e}", exc_info=True)
+        click.echo(
+            click.style(
+                f"ERROR: Redis connection failed during health check: {e}", fg="red"
+            ),
+            err=True,
+        )
         click.echo(
             click.style(
                 f"Worker Health Check: FAIL - Redis connection error: {e}", fg="red"
@@ -150,8 +181,12 @@ async def check_health_async_impl(settings_object_path: str | None = None) -> bo
         )
         return False
     except Exception as e:
-        logger.error(
-            f"An unexpected error occurred during health check: {e}", exc_info=True
+        click.echo(
+            click.style(
+                f"ERROR: An unexpected error occurred during health check: {e}",
+                fg="red",
+            ),
+            err=True,
         )
         click.echo(
             click.style(f"Worker Health Check: FAIL - Unexpected error: {e}", fg="red")
@@ -169,7 +204,7 @@ def start_rrq_worker_subprocess(
     queues: list[str] | None = None,
 ) -> subprocess.Popen | None:
     """Start an RRQ worker process, optionally for specific queues."""
-    command = ["rrq", "worker", "run"]
+    command = ["rrq", "worker", "run", "--num-workers", "1"]
 
     if settings_object_path:
         command.extend(["--settings", settings_object_path])
@@ -219,15 +254,25 @@ def terminate_worker_process(
             f"Terminating worker process group for PID {process.pid} (PGID {pgid})..."
         )
         os.killpg(pgid, signal.SIGTERM)
-        process.wait(timeout=5)
+        process.wait(timeout=10)
     except subprocess.TimeoutExpired:
-        logger.warning(
-            f"Worker process {process.pid} did not terminate gracefully (SIGTERM timeout), sending SIGKILL."
+        click.echo(
+            click.style(
+                f"WARNING: Worker process {process.pid} did not terminate gracefully (SIGTERM timeout), sending SIGKILL.",
+                fg="yellow",
+            ),
+            err=True,
         )
         with suppress(ProcessLookupError):
             os.killpg(os.getpgid(process.pid), signal.SIGKILL)
     except Exception as e:
-        logger.error(f"Unexpected error checking worker process {process.pid}: {e}")
+        click.echo(
+            click.style(
+                f"ERROR: Unexpected error checking worker process {process.pid}: {e}",
+                fg="red",
+            ),
+            err=True,
+        )
 
 
 async def watch_rrq_worker_impl(
@@ -236,9 +281,19 @@ async def watch_rrq_worker_impl(
     queues: list[str] | None = None,
 ) -> None:
     abs_watch_path = os.path.abspath(watch_path)
-    click.echo(
-        f"Watching for file changes in {abs_watch_path} to restart RRQ worker (app settings: {settings_object_path})..."
-    )
+    click.echo(f"Watching for file changes in {abs_watch_path}...")
+
+    # Load settings and display source
+    click.echo("Loading RRQ Settings... ", nl=False)
+
+    if settings_object_path:
+        click.echo(f"from --settings parameter ({settings_object_path}).")
+    elif os.getenv("RRQ_SETTINGS"):
+        click.echo(f"from RRQ_SETTINGS env var ({os.getenv('RRQ_SETTINGS')}).")
+    elif DOTENV_AVAILABLE and find_dotenv(usecwd=True):
+        click.echo("found in .env file.")
+    else:
+        click.echo("using defaults.")
     worker_process: subprocess.Popen | None = None
     loop = asyncio.get_event_loop()
     shutdown_event = asyncio.Event()
@@ -278,7 +333,9 @@ async def watch_rrq_worker_impl(
                 queues=queues,
             )
     except Exception as e:
-        logger.error(f"Error in watch_rrq_worker: {e}", exc_info=True)
+        click.echo(
+            click.style(f"ERROR: Error in watch_rrq_worker: {e}", fg="red"), err=True
+        )
     finally:
         logger.info("Exiting watch mode. Ensuring worker process is terminated.")
         if not shutdown_event.is_set():
@@ -338,28 +395,85 @@ def worker_cli():
         "The specified settings object must include a `job_registry: JobRegistry`."
     ),
 )
+@click.option(
+    "--num-workers",
+    type=int,
+    default=None,
+    help="Number of parallel worker processes to start. Defaults to the number of CPU cores.",
+)
 def worker_run_command(
     burst: bool,
     queues: tuple[str, ...],
     settings_object_path: str,
+    num_workers: int | None,
 ):
-    """Run an RRQ worker process.
+    """Run RRQ worker processes.
     Requires an application-specific settings object.
     """
-    rrq_settings = _load_app_settings(settings_object_path)
+    if num_workers is None:
+        num_workers = (
+            os.cpu_count() or 1
+        )  # Default to CPU cores, or 1 if cpu_count() is None
+        click.echo(
+            f"No --num-workers specified, defaulting to {num_workers} (CPU cores)."
+        )
+    elif num_workers <= 0:
+        click.echo(
+            click.style("ERROR: --num-workers must be a positive integer.", fg="red"),
+            err=True,
+        )
+        sys.exit(1)
 
-    # Determine queues to poll
-    queues_arg = list(queues) if queues else None
-    # Run worker in foreground (burst or continuous mode)
+    # Restrict burst mode with multiple workers
+    if num_workers > 1 and burst:
+        click.echo(
+            click.style(
+                "ERROR: --burst mode is not supported with multiple workers (--num-workers > 1). "
+                "Burst mode cannot coordinate across multiple processes.",
+                fg="red",
+            ),
+            err=True,
+        )
+        sys.exit(1)
 
-    logger.info(
-        f"Starting RRQ Worker (Burst: {burst}, App Settings: {settings_object_path})"
-    )
+    # Display settings source
+    click.echo("Loading RRQ Settings... ", nl=False)
+    if settings_object_path:
+        click.echo(f"from --settings parameter ({settings_object_path}).")
+    elif os.getenv("RRQ_SETTINGS"):
+        click.echo(f"from RRQ_SETTINGS env var ({os.getenv('RRQ_SETTINGS')}).")
+    elif DOTENV_AVAILABLE and find_dotenv(usecwd=True):
+        click.echo("found in .env file.")
+    else:
+        click.echo("using defaults.")
+
+    if num_workers == 1:
+        # Run a single worker in the current process
+        click.echo(f"Starting 1 RRQ worker process (Burst: {burst})")
+        _run_single_worker(
+            burst, list(queues) if queues else None, settings_object_path
+        )
+    else:
+        # Run multiple worker subprocesses
+        click.echo(f"Starting {num_workers} RRQ worker processes")
+        # Burst is guaranteed to be False here
+        _run_multiple_workers(
+            num_workers, list(queues) if queues else None, settings_object_path
+        )
+
+
+def _run_single_worker(
+    burst: bool,
+    queues_arg: list[str] | None,
+    settings_object_path: str | None,
+):
+    """Helper function to run a single RRQ worker instance."""
+    rrq_settings = _load_app_settings(settings_object_path)
 
     if not rrq_settings.job_registry:
         click.echo(
             click.style(
-                "ERROR: No 'job_registry_app'. You must provide a JobRegistry instance in settings.",
+                "ERROR: No 'job_registry'. You must provide a JobRegistry instance in settings.",
                 fg="red",
             ),
             err=True,
@@ -378,22 +492,182 @@ def worker_run_command(
         burst=burst,
     )
 
-    loop = asyncio.get_event_loop()
     try:
-        logger.info("Starting worker run loop...")
-        loop.run_until_complete(worker_instance.run())
+        logger.info("Starting worker run loop for single worker...")
+        asyncio.run(worker_instance.run())
     except KeyboardInterrupt:
         logger.info("RRQ Worker run interrupted by user (KeyboardInterrupt).")
     except Exception as e:
-        logger.error(f"Exception during RRQ Worker run: {e}", exc_info=True)
+        click.echo(
+            click.style(f"ERROR: Exception during RRQ Worker run: {e}", fg="red"),
+            err=True,
+        )
+        # Consider re-raising or sys.exit(1) if the exception means failure
     finally:
-        logger.info("RRQ Worker run finished or exited. Cleaning up event loop.")
-        if loop.is_running():
-            loop.run_until_complete(loop.shutdown_asyncgens())
-            loop.close()
+        # asyncio.run handles loop cleanup.
+        logger.info("RRQ Worker run finished or exited.")
         logger.info("RRQ Worker has shut down.")
 
 
+def _run_multiple_workers(
+    num_workers: int,
+    queues: list[str] | None,
+    settings_object_path: str | None,
+):
+    """Manages multiple worker subprocesses."""
+    processes: list[subprocess.Popen] = []
+    # loop = asyncio.get_event_loop() # Not needed here, this function is synchronous
+
+    original_sigint_handler = signal.getsignal(signal.SIGINT)
+    original_sigterm_handler = signal.getsignal(signal.SIGTERM)
+
+    def sig_handler(signum, frame):
+        click.echo(
+            f"\nSignal {signal.Signals(signum).name} received. Terminating child workers..."
+        )
+        # Send SIGTERM to all processes
+        for i, p in enumerate(processes):
+            if p.poll() is None:  # Process is still running
+                try:
+                    pgid = os.getpgid(p.pid)
+                    click.echo(f"Sending SIGTERM to worker {i + 1} (PID {p.pid})...")
+                    os.killpg(pgid, signal.SIGTERM)
+                except (ProcessLookupError, OSError):
+                    pass  # Process already dead
+        # Restore original handlers before exiting or re-raising
+        signal.signal(signal.SIGINT, original_sigint_handler)
+        signal.signal(signal.SIGTERM, original_sigterm_handler)
+        # Propagate signal to ensure parent exits if it was, e.g., a Ctrl+C
+        # This is a bit tricky; for now, just exit.
+        # A more robust way might involve re-raising the signal if not handled by click/asyncio.
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, sig_handler)
+    signal.signal(signal.SIGTERM, sig_handler)
+
+    try:
+        for i in range(num_workers):
+            # Construct the command for the subprocess.
+            # Each subprocess runs 'rrq worker run' for a single worker.
+            # We pass along relevant flags like --settings, --queue, and --burst.
+            # Crucially, we do *not* pass --num-workers to the child,
+            # or rather, we could conceptually pass --num-workers 1.
+            # Use the rrq executable from the same venv
+            venv_bin_dir = os.path.dirname(sys.executable)
+            rrq_executable = os.path.join(venv_bin_dir, "rrq")
+            cmd = [rrq_executable, "worker", "run", "--num-workers=1"]
+            if settings_object_path:
+                cmd.extend(["--settings", settings_object_path])
+            elif os.getenv("RRQ_SETTINGS"):
+                # Pass the RRQ_SETTINGS env var as explicit parameter to subprocess
+                cmd.extend(["--settings", os.getenv("RRQ_SETTINGS")])
+            else:
+                # Default to app.config.rrq.rrq_settings for ResQ
+                cmd.extend(["--settings", "app.config.rrq.rrq_settings"])
+            if queues:
+                for q_name in queues:
+                    cmd.extend(["--queue", q_name])
+            click.echo(f"Starting worker subprocess {i + 1}/{num_workers}...")
+
+            # Set up environment - add current directory to PYTHONPATH
+            env = os.environ.copy()
+            current_pythonpath = env.get("PYTHONPATH", "")
+            current_dir = os.getcwd()
+            if current_pythonpath:
+                env["PYTHONPATH"] = f"{current_dir}:{current_pythonpath}"
+            else:
+                env["PYTHONPATH"] = current_dir
+
+            # Configure output redirection
+            is_testing = "PYTEST_CURRENT_TEST" in os.environ
+            stdout_dest = None if not is_testing else subprocess.DEVNULL
+            stderr_dest = None if not is_testing else subprocess.DEVNULL
+
+            process = subprocess.Popen(
+                cmd,
+                start_new_session=True,
+                stdout=stdout_dest,
+                stderr=stderr_dest,
+                cwd=os.getcwd(),
+                env=env,
+            )
+            processes.append(process)
+            click.echo(f"Worker subprocess {i + 1} started with PID {process.pid}")
+
+        # Wait for all processes to complete
+        click.echo(f"All {num_workers} workers started. Press Ctrl+C to stop.")
+        exit_codes = []
+
+        try:
+            for p in processes:
+                exit_code = p.wait()
+                exit_codes.append(exit_code)
+        except KeyboardInterrupt:
+            # Signal handler has already sent SIGTERM, now wait with timeout
+            max_wait = 10
+            check_interval = 0.1
+            elapsed = 0
+
+            while elapsed < max_wait:
+                time.sleep(check_interval)
+                elapsed += check_interval
+
+                # Check if all processes have terminated
+                all_terminated = all(p.poll() is not None for p in processes)
+                if all_terminated:
+                    click.echo("All workers terminated gracefully.")
+                    break
+            else:
+                # Timeout reached, force kill remaining processes
+                for i, p in enumerate(processes):
+                    if p.poll() is None:
+                        try:
+                            click.echo(
+                                click.style(
+                                    f"WARNING: Worker {i + 1} did not terminate gracefully, sending SIGKILL.",
+                                    fg="yellow",
+                                ),
+                                err=True,
+                            )
+                            os.killpg(os.getpgid(p.pid), signal.SIGKILL)
+                        except (ProcessLookupError, OSError):
+                            pass
+
+            # Collect exit codes
+            for p in processes:
+                exit_codes.append(p.wait())
+
+        # Report results
+        for i, exit_code in enumerate(exit_codes):
+            click.echo(f"Worker subprocess {i + 1} exited with code {exit_code}")
+            if exit_code != 0:
+                click.echo(
+                    click.style(
+                        f"Worker subprocess {i + 1} failed with exit code {exit_code}",
+                        fg="red",
+                    ),
+                    err=True,
+                )
+
+    except Exception as e:
+        click.echo(
+            click.style(f"ERROR: Error managing worker subprocesses: {e}", fg="red"),
+            err=True,
+        )
+        # Terminate any running processes if an error occurs in the manager
+        for p in processes:
+            if p.poll() is None:  # If process is still running
+                terminate_worker_process(p, logger)
+    finally:
+        logger.info("All worker subprocesses terminated or completed.")
+        # Restore original signal handlers
+        signal.signal(signal.SIGINT, original_sigint_handler)
+        signal.signal(signal.SIGTERM, original_sigterm_handler)
+        # Any other cleanup for the parent process
+        # No loop to check or close here as this part is synchronous
+        logger.info("Parent process for multi-worker management is exiting.")
+
+
 @worker_cli.command("watch")
 @click.option(
     "--path",

rrq/cron.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any, Optional, Sequence
+
+from pydantic import BaseModel, Field, PrivateAttr
+
+MONTH_NAMES = {
+    "jan": 1,
+    "feb": 2,
+    "mar": 3,
+    "apr": 4,
+    "may": 5,
+    "jun": 6,
+    "jul": 7,
+    "aug": 8,
+    "sep": 9,
+    "oct": 10,
+    "nov": 11,
+    "dec": 12,
+}
+
+WEEKDAY_NAMES = {
+    "sun": 0,
+    "mon": 1,
+    "tue": 2,
+    "wed": 3,
+    "thu": 4,
+    "fri": 5,
+    "sat": 6,
+}
+
+
+def _parse_value(value: str, names: dict[str, int], min_val: int, max_val: int) -> int:
+    if value.lower() in names:
+        return names[value.lower()]
+    num = int(value)
+    if names is WEEKDAY_NAMES and num == 7:
+        num = 0
+    if not (min_val <= num <= max_val):
+        raise ValueError(f"value {num} out of range {min_val}-{max_val}")
+    return num
+
+
+def _parse_field(
+    field: str, *, names: dict[str, int] | None, min_val: int, max_val: int
+) -> Sequence[int]:
+    names = names or {}
+    if field == "*":
+        return list(range(min_val, max_val + 1))
+    values: set[int] = set()
+    for part in field.split(","):
+        step = 1
+        if "/" in part:
+            base, step_str = part.split("/", 1)
+            step = int(step_str)
+        else:
+            base = part
+        if base == "*":
+            start, end = min_val, max_val
+        elif "-" in base:
+            a, b = base.split("-", 1)
+            start = _parse_value(a, names, min_val, max_val)
+            end = _parse_value(b, names, min_val, max_val)
+        else:
+            val = _parse_value(base, names, min_val, max_val)
+            start = end = val
+        if start > end:
+            raise ValueError(f"invalid range {base}")
+        for v in range(start, end + 1, step):
+            values.add(v)
+    return sorted(values)
+
+
+class CronSchedule:
+    """Represents a cron schedule expression."""
+
+    def __init__(self, expression: str) -> None:
+        fields = expression.split()
+        if len(fields) != 5:
+            raise ValueError("Cron expression must have 5 fields")
+        minute, hour, dom, month, dow = fields
+        self.minutes = _parse_field(minute, names=None, min_val=0, max_val=59)
+        self.hours = _parse_field(hour, names=None, min_val=0, max_val=23)
+        self.dom = _parse_field(dom, names=None, min_val=1, max_val=31)
+        self.months = _parse_field(month, names=MONTH_NAMES, min_val=1, max_val=12)
+        self.dow = _parse_field(dow, names=WEEKDAY_NAMES, min_val=0, max_val=6)
+        self.dom_all = dom == "*"
+        self.dow_all = dow == "*"
+
+    def next_after(self, dt: datetime) -> datetime:
+        dt = dt.replace(second=0, microsecond=0) + timedelta(minutes=1)
+        while True:
+            if dt.month not in self.months:
+                dt += timedelta(minutes=1)
+                continue
+            if dt.hour not in self.hours or dt.minute not in self.minutes:
+                dt += timedelta(minutes=1)
+                continue
+            dom_match = dt.day in self.dom
+            # Convert Python weekday (Monday=0) to cron weekday (Sunday=0)
+            # Python: Mon=0, Tue=1, Wed=2, Thu=3, Fri=4, Sat=5, Sun=6
+            # Cron:   Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6
+            python_weekday = dt.weekday()
+            cron_weekday = (python_weekday + 1) % 7
+            dow_match = cron_weekday in self.dow
+
+            if self.dom_all and self.dow_all:
+                condition = True
+            elif self.dom_all:
+                # Only day-of-week constraint
+                condition = dow_match
+            elif self.dow_all:
+                # Only day-of-month constraint
+                condition = dom_match
+            else:
+                # Both constraints specified - use OR logic (standard cron behavior)
+                condition = dom_match or dow_match
+            if condition:
+                return dt
+            dt += timedelta(minutes=1)
+
+
+class CronJob(BaseModel):
+    """Simple cron job specification based on a cron schedule."""
+
+    function_name: str
+    schedule: str = Field(
+        description="Cron expression 'm h dom mon dow'. Resolution is one minute."
+    )
+    args: list[Any] = Field(default_factory=list)
+    kwargs: dict[str, Any] = Field(default_factory=dict)
+    queue_name: Optional[str] = None
+    unique: bool = False
+
+    # Next run time and parsed schedule are maintained at runtime
+    next_run_time: Optional[datetime] = Field(default=None, exclude=True)
+    _cron: CronSchedule | None = PrivateAttr(default=None)
+
+    def model_post_init(self, __context: Any) -> None:  # type: ignore[override]
+        self._cron = CronSchedule(self.schedule)
+
+    def schedule_next(self, now: Optional[datetime] = None) -> None:
+        """Compute the next run time strictly after *now*."""
+        now = (now or datetime.now(UTC)).replace(second=0, microsecond=0)
+        if self._cron is None:
+            self._cron = CronSchedule(self.schedule)
+        self.next_run_time = self._cron.next_after(now)
+
+    def due(self, now: Optional[datetime] = None) -> bool:
+        now = now or datetime.now(UTC)
+        if self.next_run_time is None:
+            self.schedule_next(now)
+        return now >= (self.next_run_time or now)

rrq/settings.py

@@ -21,6 +21,7 @@ from .constants import (
     DEFAULT_UNIQUE_JOB_LOCK_TTL_SECONDS,
 )
 from .registry import JobRegistry
+from .cron import CronJob
 
 
 class RRQSettings(BaseSettings):
@@ -97,6 +98,10 @@ class RRQSettings(BaseSettings):
         default=None,
         description="Job registry instance, typically provided by the application.",
     )
+    cron_jobs: list[CronJob] = Field(
+        default_factory=list,
+        description="Optional list of cron job specifications to run periodically.",
+    )
     model_config = SettingsConfigDict(
         env_prefix="RRQ_",
         extra="ignore",