toro-queue 0.1.0__py3-none-any.whl

toro/__init__.py

@@ -0,0 +1,9 @@
+"""toro — an async-first, Redis-backed job queue for Python."""
+
+from .errors import JobFailedError, ToroError
+from .job import Job, JobOptions, JobState
+from .queue import Queue
+from .worker import Worker
+
+__all__ = ["Job", "JobFailedError", "JobOptions", "JobState", "Queue", "ToroError", "Worker"]
+__version__ = "0.0.1"

toro/connection.py

@@ -0,0 +1,31 @@
+"""Redis connection factory with sane defaults for toro's long-lived clients."""
+
+from __future__ import annotations
+
+import redis.asyncio as aioredis
+from redis.asyncio.retry import Retry
+from redis.backoff import ExponentialBackoff
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+
+
+def connect(url: str) -> aioredis.Redis:
+    """Open a ``decode_responses`` client tuned for toro's long-lived connections.
+
+    Connections that sit idle for a while (a worker's blocking pop, the result()
+    pub/sub, mostly-idle producers) get:
+
+    * ``health_check_interval`` + ``socket_keepalive`` to recycle half-open
+      connections a NAT/load-balancer idle timeout silently dropped, instead of
+      failing the next real command.
+    * a ``Retry`` policy so a transient reconnect is invisible to the worker loops,
+      rather than surfacing as an exception they'd swallow into a skipped iteration.
+    """
+    return aioredis.from_url(
+        url,
+        decode_responses=True,
+        health_check_interval=30,
+        socket_keepalive=True,
+        retry=Retry(ExponentialBackoff(), retries=3),
+        retry_on_error=[RedisConnectionError, RedisTimeoutError],
+    )

toro/errors.py

@@ -0,0 +1,15 @@
+"""toro exceptions."""
+
+from __future__ import annotations
+
+
+class ToroError(Exception):
+    """Base class for toro errors."""
+
+
+class JobFailedError(ToroError):
+    """Raised by result() when the awaited job ended in the failed state."""
+
+    def __init__(self, reason: str | None) -> None:
+        super().__init__(reason or "job failed")
+        self.reason = reason

toro/job.py

@@ -0,0 +1,154 @@
+"""The Job: a typed view over the Redis hash that stores one unit of work."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Literal, Protocol, cast
+
+from redis.asyncio import Redis
+
+# The lifecycle states a job can be in (also the queryable states for get_jobs).
+JobState = Literal["wait", "active", "delayed", "completed", "failed"]
+
+
+class SupportsResult(Protocol):
+    """The slice of Queue that `Job.result()` needs. Typing `_queue` against this
+    (not the concrete Queue) keeps the domain object from importing the queue/redis
+    layer — dependency inversion, and no circular import to dodge.
+    """
+
+    async def result(self, job_id: str, *, timeout: float = ...) -> Any: ...
+
+
+@dataclass
+class JobOptions:
+    """Per-job options (delay, attempts, backoff, priority, auto-removal)."""
+
+    delay: int = 0  # ms to wait before the job becomes processable
+    attempts: int = 1  # total tries before the job is considered failed
+    backoff: Any = None  # int ms, or {"type": "fixed"|"exponential", "delay": ms}
+    priority: int = 0  # higher = more urgent (global order); 0 = default, FIFO
+    # Auto-removal: None/False keep all · True remove on finish · int keep last N ·
+    # {"count": N, "age": seconds} keep within count and/or age.
+    remove_on_complete: Any = None
+    remove_on_fail: Any = None
+
+    def to_dict(self) -> dict:
+        return {
+            "delay": self.delay,
+            "attempts": self.attempts,
+            "backoff": self.backoff,
+            "priority": self.priority,
+            "removeOnComplete": self.remove_on_complete,
+            "removeOnFail": self.remove_on_fail,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict) -> JobOptions:
+        return cls(
+            delay=d.get("delay", 0),
+            attempts=d.get("attempts", 1),
+            backoff=d.get("backoff"),
+            priority=d.get("priority", 0),
+            remove_on_complete=d.get("removeOnComplete"),
+            remove_on_fail=d.get("removeOnFail"),
+        )
+
+    @staticmethod
+    def keep_args(opt: Any) -> tuple[int, int]:
+        """Map a remove option to (keepCount, keepAge_seconds) for the Lua side.
+
+        keepCount: -1 keep all · 0 remove immediately · N keep newest N.
+        keepAge:   -1 no age limit · S keep only those finished within S seconds.
+        """
+        if opt is None or opt is False:
+            return (-1, -1)
+        if opt is True:
+            return (0, -1)
+        if isinstance(opt, int):
+            return (int(opt), -1)
+        if isinstance(opt, dict):
+            return (int(opt.get("count", -1)), int(opt.get("age", -1)))
+        return (-1, -1)
+
+
+@dataclass(frozen=True, slots=True)
+class JobContext:
+    """Worker-side handles attached to a Job while its processor runs, so the handler
+    can report progress / append logs.
+    """
+
+    redis: Redis
+    job_key: str
+    events_key: str
+    logs_key: str
+    job_id: str
+
+
+@dataclass
+class Job:
+    """A snapshot of one job: its id, data, options, state and lifecycle timestamps."""
+
+    id: str
+    name: str
+    data: Any
+    opts: JobOptions = field(default_factory=JobOptions)
+    attempts_made: int = 0
+    timestamp: int | None = None
+    returnvalue: Any = None
+    failed_reason: str | None = None
+    state: JobState | None = None
+    processed_on: int | None = None
+    finished_on: int | None = None
+    progress: Any = None
+    stacktrace: str | None = None
+    # Back-reference to the owning Queue, set on jobs returned by Queue.add() so
+    # producers can `await job.result()`. Not part of the job's data/identity.
+    _queue: SupportsResult | None = field(default=None, repr=False, compare=False)
+    # Worker-side context, set while a processor runs so the handler can report
+    # progress and append logs.
+    _ctx: JobContext | None = field(default=None, repr=False, compare=False)
+
+    async def result(self, *, timeout: float = 30.0) -> Any:
+        """Wait for this job to finish; return its value or raise JobFailedError."""
+        if self._queue is None:
+            raise RuntimeError("job.result() requires a job returned by Queue.add()")
+        return await self._queue.result(self.id, timeout=timeout)
+
+    async def update_progress(self, value: Any) -> None:
+        """Report progress (a number 0-100 or any JSON value) from a processor."""
+        if self._ctx is None:
+            raise RuntimeError("update_progress() is only available inside a worker processor")
+        ctx = self._ctx
+        self.progress = value
+        await ctx.redis.hset(ctx.job_key, "progress", json.dumps(value))
+        await ctx.redis.publish(
+            ctx.events_key,
+            json.dumps({"jobId": ctx.job_id, "event": "progress", "progress": value}),
+        )
+
+    async def log(self, message: str) -> None:
+        """Append a log line to this job (visible in the dashboard)."""
+        if self._ctx is None:
+            raise RuntimeError("log() is only available inside a worker processor")
+        await self._ctx.redis.rpush(self._ctx.logs_key, message)
+
+    @classmethod
+    def from_hash(cls, job_id: str, h: dict) -> Job:
+        """Build a Job from a decoded Redis hash (str keys/values)."""
+        return cls(
+            id=job_id,
+            name=h.get("name", ""),
+            data=json.loads(h["data"]) if h.get("data") else None,
+            opts=JobOptions.from_dict(json.loads(h["opts"])) if h.get("opts") else JobOptions(),
+            attempts_made=int(h.get("attemptsMade", 0)),
+            timestamp=int(h["timestamp"]) if h.get("timestamp") else None,
+            returnvalue=json.loads(h["returnvalue"]) if h.get("returnvalue") else None,
+            failed_reason=h.get("failedReason"),
+            state=cast("JobState | None", h.get("state")),  # Redis stores it untyped
+            processed_on=int(h["processedOn"]) if h.get("processedOn") else None,
+            finished_on=int(h["finishedOn"]) if h.get("finishedOn") else None,
+            progress=json.loads(h["progress"]) if h.get("progress") else None,
+            stacktrace=h.get("stacktrace"),
+        )

toro/keys.py

@@ -0,0 +1,108 @@
+"""Central place that knows how Redis keys are laid out for a queue.
+
+Keeping this in one spot means the Lua scripts and the Python side can never
+disagree about where a list/zset/hash lives.
+"""
+
+from __future__ import annotations
+
+
+class Keys:
+    """Computes the Redis key names for one queue from its prefix + name."""
+
+    def __init__(self, queue_name: str, prefix: str = "toro") -> None:
+        self.queue_name = queue_name
+        self.prefix = prefix
+        self.base = f"{prefix}:{queue_name}:"
+
+    @property
+    def id(self) -> str:
+        return f"{self.base}id"
+
+    @property
+    def prioritized(self) -> str:
+        # The single global-priority-ordered store of waiting jobs.
+        return f"{self.base}prioritized"
+
+    @property
+    def marker(self) -> str:
+        # Wakeup signal: idle workers BZPOPMIN here; producers ZADD an idempotent base marker.
+        return f"{self.base}marker"
+
+    @property
+    def pc(self) -> str:
+        # Priority sequence counter — breaks priority ties in FIFO order.
+        return f"{self.base}pc"
+
+    @property
+    def active(self) -> str:
+        return f"{self.base}active"
+
+    @property
+    def delayed(self) -> str:
+        return f"{self.base}delayed"
+
+    @property
+    def completed(self) -> str:
+        return f"{self.base}completed"
+
+    @property
+    def failed(self) -> str:
+        return f"{self.base}failed"
+
+    @property
+    def meta_paused(self) -> str:
+        # Existence flag: when set, workers stop claiming new jobs.
+        return f"{self.base}meta-paused"
+
+    @property
+    def events(self) -> str:
+        # Pub/sub channel for job outcomes (completed/failed) — drives result() and live UI.
+        return f"{self.base}events"
+
+    @property
+    def limiter(self) -> str:
+        # Token-bucket hash {tokens, ts} — the queue-wide rate limit, shared by all workers.
+        return f"{self.base}limiter"
+
+    @property
+    def stalled(self) -> str:
+        return f"{self.base}stalled"
+
+    @property
+    def stalled_check(self) -> str:
+        return f"{self.base}stalled-check"
+
+    @property
+    def repeat(self) -> str:
+        # ZSET of scheduler ids -> next-run timestamp.
+        return f"{self.base}repeat"
+
+    def scheduler(self, scheduler_id: str) -> str:
+        # HASH holding a scheduler's template (name, every/cron, data, opts).
+        return f"{self.base}repeat:{scheduler_id}"
+
+    @property
+    def workers(self) -> str:
+        # ZSET of live worker ids -> last-heartbeat timestamp (ms). Stale entries
+        # are pruned lazily on read; powers the dashboard's "who's running" view.
+        return f"{self.base}workers"
+
+    def worker(self, worker_id: str) -> str:
+        # HASH with a worker's presence record (host, pid, concurrency, counts, ...).
+        return f"{self.base}worker:{worker_id}"
+
+    @property
+    def departed(self) -> str:
+        # Capped LIST of recent worker departures: graceful stop ("stopped") or a
+        # lost heartbeat ("lost" = crashed/killed). Gives the dashboard death history.
+        return f"{self.base}departed"
+
+    def job(self, job_id: str | int) -> str:
+        return f"{self.base}{job_id}"
+
+    def lock(self, job_id: str | int) -> str:
+        return f"{self.base}{job_id}:lock"
+
+    def logs(self, job_id: str | int) -> str:
+        return f"{self.base}{job_id}:logs"