baqueue 0.1.0__py3-none-any.whl

baqueue/events.py

@@ -0,0 +1,64 @@
+"""Event bus for BaQueue lifecycle events."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import defaultdict
+from typing import Any, Callable, Coroutine
+
+logger = logging.getLogger("baqueue.events")
+
+EventHandler = Callable[..., Coroutine[Any, Any, None]]
+
+
+class EventBus:
+    """Simple async event bus for job lifecycle events.
+
+    Built-in events:
+        job.pushed, job.started, job.completed, job.failed, job.retrying,
+        batch.completed, batch.failed,
+        worker.started, worker.stopped,
+        supervisor.started, supervisor.stopped,
+        queue.pruned
+    """
+
+    _handlers: dict[str, list[EventHandler]]
+    _instance: EventBus | None = None
+
+    def __init__(self) -> None:
+        self._handlers = defaultdict(list)
+
+    @classmethod
+    def default(cls) -> EventBus:
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def on(self, event: str, handler: EventHandler) -> None:
+        self._handlers[event].append(handler)
+
+    def off(self, event: str, handler: EventHandler | None = None) -> None:
+        if handler is None:
+            self._handlers.pop(event, None)
+        else:
+            handlers = self._handlers.get(event, [])
+            self._handlers[event] = [h for h in handlers if h is not handler]
+
+    async def emit(self, event: str, **kwargs: Any) -> None:
+        for handler in self._handlers.get(event, []):
+            try:
+                await handler(**kwargs)
+            except Exception:
+                logger.exception("Error in event handler for %s", event)
+
+    def emit_nowait(self, event: str, **kwargs: Any) -> None:
+        """Fire-and-forget emit for use in sync contexts."""
+        try:
+            loop = asyncio.get_running_loop()
+            loop.create_task(self.emit(event, **kwargs))
+        except RuntimeError:
+            pass
+
+    def clear(self) -> None:
+        self._handlers.clear()

baqueue/job.py

@@ -0,0 +1,128 @@
+"""Base Job class and decorator for defining queue jobs."""
+
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Coroutine
+
+from baqueue.serializer import JobPayload, get_class_path
+
+
+class Job:
+    """Base class for queue jobs.
+
+    Usage::
+
+        class SendEmail(Job):
+            queue = "emails"
+            max_attempts = 3
+            backoff = "exponential"
+            timeout = 30
+            tags = ["email"]
+
+            async def handle(self, to: str, subject: str, body: str):
+                await send_email(to, subject, body)
+
+            async def on_failure(self, error: Exception, payload: JobPayload):
+                log.error("Email send failed: %s", error)
+    """
+
+    queue: str = "default"
+    max_attempts: int = 3
+    backoff: str | list[int] = "exponential"
+    timeout: int = 60
+    tags: list[str] = []
+
+    async def handle(self, **kwargs: Any) -> Any:
+        raise NotImplementedError("Subclasses must implement handle()")
+
+    async def on_failure(self, error: Exception, payload: JobPayload) -> None:
+        """Called when the job fails all retry attempts."""
+        pass
+
+    async def on_success(self, result: Any, payload: JobPayload) -> None:
+        """Called when the job completes successfully."""
+        pass
+
+    def build_payload(self, **kwargs: Any) -> JobPayload:
+        return JobPayload(
+            job_class=get_class_path(self.__class__),
+            data=kwargs,
+            queue=self.queue,
+            max_attempts=self.max_attempts,
+            backoff=self.backoff,
+            timeout=self.timeout,
+            tags=list(self.tags),
+        )
+
+    @classmethod
+    def as_job(
+        cls,
+        queue: str = "default",
+        max_attempts: int = 3,
+        backoff: str | list[int] = "exponential",
+        timeout: int = 60,
+        tags: list[str] | None = None,
+    ) -> Callable:
+        """Decorator to turn an async function into a dispatchable job.
+
+        Usage::
+
+            @Job.as_job(queue="emails", max_attempts=3)
+            async def send_email(to, subject, body):
+                ...
+
+            await Queue.push(send_email, to="a@b.com", subject="Hi", body="Hello")
+        """
+        def decorator(func: Callable[..., Coroutine]) -> FunctionJob:
+            job = FunctionJob(
+                func=func,
+                queue=queue,
+                max_attempts=max_attempts,
+                backoff=backoff,
+                timeout=timeout,
+                tags=tags or [],
+            )
+            functools.update_wrapper(job, func)
+            return job
+        return decorator
+
+
+class FunctionJob(Job):
+    """Wraps a plain async function as a Job."""
+
+    def __init__(
+        self,
+        func: Callable[..., Coroutine],
+        queue: str = "default",
+        max_attempts: int = 3,
+        backoff: str | list[int] = "exponential",
+        timeout: int = 60,
+        tags: list[str] | None = None,
+    ):
+        self._func = func
+        self.queue = queue
+        self.max_attempts = max_attempts
+        self.backoff = backoff
+        self.timeout = timeout
+        self.tags = tags or []
+        self.__name__ = func.__name__
+        self.__qualname__ = func.__qualname__
+        self.__module__ = func.__module__
+
+    async def handle(self, **kwargs: Any) -> Any:
+        return await self._func(**kwargs)
+
+    def build_payload(self, **kwargs: Any) -> JobPayload:
+        return JobPayload(
+            job_class=f"{self._func.__module__}.{self._func.__qualname__}",
+            data=kwargs,
+            queue=self.queue,
+            max_attempts=self.max_attempts,
+            backoff=self.backoff,
+            timeout=self.timeout,
+            tags=list(self.tags),
+        )
+
+    async def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        return await self._func(*args, **kwargs)

baqueue/pruner.py

@@ -0,0 +1,128 @@
+"""Pruner - removes old or unwanted jobs from the queue system."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from baqueue.config import BaQueueConfig
+from baqueue.drivers.base import BaseDriver
+from baqueue.events import EventBus
+
+logger = logging.getLogger("baqueue.pruner")
+
+
+class Pruner:
+    """Automatically prunes old jobs on a schedule.
+
+    Can also be triggered manually via the CLI or API.
+    """
+
+    def __init__(
+        self,
+        driver: BaseDriver,
+        config: BaQueueConfig | None = None,
+        events: EventBus | None = None,
+    ):
+        self.driver = driver
+        self.config = config or BaQueueConfig()
+        self.events = events or EventBus.default()
+        self._running = False
+
+    # ── Threshold resolution ────────────────────────────────────
+    # New *_seconds fields are primary. The legacy *_hours fields override
+    # when set to a positive value, so existing JSON configs keep working.
+
+    @property
+    def completed_threshold(self) -> float:
+        if self.config.prune_completed_hours > 0:
+            return self.config.prune_completed_hours * 3600
+        return float(self.config.prune_completed_seconds)
+
+    @property
+    def failed_threshold(self) -> float:
+        if self.config.prune_failed_hours > 0:
+            return self.config.prune_failed_hours * 3600
+        return float(self.config.prune_other_seconds)
+
+    @property
+    def cancelled_threshold(self) -> float:
+        if self.config.prune_cancelled_hours > 0:
+            return self.config.prune_cancelled_hours * 3600
+        return float(self.config.prune_other_seconds)
+
+    @property
+    def metrics_threshold(self) -> float:
+        if self.config.prune_metrics_hours > 0:
+            return self.config.prune_metrics_hours * 3600
+        return float(self.config.prune_metrics_seconds)
+
+    async def prune_once(self) -> dict[str, int]:
+        """Run a single prune pass based on config."""
+        results: dict[str, int] = {}
+
+        if self.completed_threshold > 0:
+            results["completed"] = await self.driver.prune(
+                status="completed",
+                older_than_seconds=self.completed_threshold,
+            )
+
+        if self.failed_threshold > 0:
+            results["failed"] = await self.driver.prune(
+                status="failed",
+                older_than_seconds=self.failed_threshold,
+            )
+
+        if self.cancelled_threshold > 0:
+            results["cancelled"] = await self.driver.prune(
+                status="cancelled",
+                older_than_seconds=self.cancelled_threshold,
+            )
+
+        if self.metrics_threshold > 0:
+            results["metrics"] = await self.driver.prune_metrics(
+                older_than_seconds=self.metrics_threshold,
+            )
+
+        total = sum(results.values())
+        if total > 0:
+            logger.info("Pruned %d entries: %s", total, results)
+            self.events.emit_nowait("queue.pruned", results=results)
+
+        return results
+
+    async def prune_by_tag(self, tag: str) -> int:
+        count = await self.driver.prune(tag=tag)
+        if count > 0:
+            logger.info("Pruned %d jobs with tag '%s'", count, tag)
+        return count
+
+    async def prune_by_status(self, status: str, hours: float | None = None) -> int:
+        older_than = hours * 3600 if hours else None
+        count = await self.driver.prune(status=status, older_than_seconds=older_than)
+        if count > 0:
+            logger.info("Pruned %d jobs with status '%s'", count, status)
+        return count
+
+    async def start(self, interval_seconds: float | None = None) -> None:
+        """Start automatic pruning loop. Uses config.prune_interval_seconds by default."""
+        if interval_seconds is None:
+            interval_seconds = float(self.config.prune_interval_seconds)
+        interval_seconds = max(1.0, float(interval_seconds))
+
+        self._running = True
+        logger.info("Pruner started (every %.0fs)", interval_seconds)
+
+        try:
+            while self._running:
+                try:
+                    await self.prune_once()
+                except Exception:
+                    logger.exception("Error during prune cycle")
+                await asyncio.sleep(interval_seconds)
+        except asyncio.CancelledError:
+            self._running = False
+            raise
+
+    def stop(self) -> None:
+        self._running = False

baqueue/queue.py

@@ -0,0 +1,225 @@
+"""Queue manager - the main entry point for dispatching jobs."""
+
+from __future__ import annotations
+
+from typing import Any, Type
+
+from baqueue.config import BaQueueConfig
+from baqueue.drivers.base import BaseDriver
+from baqueue.drivers.memory_driver import MemoryDriver
+from baqueue.events import EventBus
+from baqueue.job import Job, FunctionJob
+from baqueue.serializer import JobPayload, _now_ts
+
+
+class Queue:
+    """Central queue manager. Maintains a singleton driver connection.
+
+    Usage::
+
+        Queue.configure(BaQueueConfig(driver=DriverConfig(name="redis", url="redis://localhost")))
+        await Queue.push(SendEmail, to="user@example.com", subject="Hi", body="Hello")
+    """
+
+    _driver: BaseDriver | None = None
+    _config: BaQueueConfig | None = None
+    _events: EventBus | None = None
+
+    @classmethod
+    def configure(cls, config: BaQueueConfig | None = None, driver: BaseDriver | None = None) -> None:
+        cls._config = config or BaQueueConfig()
+        if driver is not None:
+            driver.auto_cleanup_on_disk_full = cls._config.auto_cleanup_on_disk_full
+        cls._driver = driver
+        cls._events = EventBus.default()
+
+    @classmethod
+    def get_driver(cls) -> BaseDriver:
+        if cls._driver is None:
+            cls._driver = _create_driver(cls._config or BaQueueConfig())
+        return cls._driver
+
+    @classmethod
+    def get_config(cls) -> BaQueueConfig:
+        if cls._config is None:
+            cls._config = BaQueueConfig()
+        return cls._config
+
+    @classmethod
+    def get_events(cls) -> EventBus:
+        if cls._events is None:
+            cls._events = EventBus.default()
+        return cls._events
+
+    @classmethod
+    async def connect(cls) -> None:
+        await cls.get_driver().connect()
+
+    @classmethod
+    async def disconnect(cls) -> None:
+        driver = cls.get_driver()
+        await driver.disconnect()
+        cls._driver = None
+
+    # ── Dispatch ────────────────────────────────────────────────
+
+    @classmethod
+    async def push(cls, job: Type[Job] | Job | FunctionJob, **kwargs: Any) -> str:
+        """Push a job onto the queue for immediate processing."""
+        payload = _build_payload(job, kwargs)
+        job_id = await cls.get_driver().push(payload)
+        cls.get_events().emit_nowait("job.pushed", payload=payload)
+        return job_id
+
+    @classmethod
+    async def later(
+        cls,
+        job: Type[Job] | Job | FunctionJob,
+        delay: float,
+        **kwargs: Any,
+    ) -> str:
+        """Push a job with a delay (in seconds)."""
+        payload = _build_payload(job, kwargs)
+        payload.delay_until = _now_ts() + delay
+        job_id = await cls.get_driver().push(payload)
+        cls.get_events().emit_nowait("job.pushed", payload=payload)
+        return job_id
+
+    @classmethod
+    async def bulk(cls, jobs: list[tuple[Type[Job] | Job | FunctionJob, dict[str, Any]]]) -> list[str]:
+        """Push multiple jobs at once."""
+        payloads = [_build_payload(j, kw) for j, kw in jobs]
+        ids = await cls.get_driver().push_many(payloads)
+        for p in payloads:
+            cls.get_events().emit_nowait("job.pushed", payload=p)
+        return ids
+
+    # ── Query ───────────────────────────────────────────────────
+
+    @classmethod
+    async def size(cls, queue: str = "default") -> int:
+        return await cls.get_driver().size(queue)
+
+    @classmethod
+    async def queues(cls) -> list[str]:
+        return await cls.get_driver().queues()
+
+    @classmethod
+    async def get_job(cls, job_id: str) -> JobPayload | None:
+        return await cls.get_driver().get_job(job_id)
+
+    @classmethod
+    async def get_jobs(cls, **kwargs: Any) -> list[JobPayload]:
+        return await cls.get_driver().get_jobs(**kwargs)
+
+    # ── Retry ───────────────────────────────────────────────────
+
+    @classmethod
+    async def retry_failed(
+        cls,
+        queue: str | None = None,
+        tag: str | None = None,
+        created_from: float | None = None,
+        created_to: float | None = None,
+    ) -> int:
+        """Retry all failed jobs matching the optional filters. Returns count retried.
+
+        Each matched job is released back onto its queue with ``delay=0`` —
+        the same path used by single-job retry.
+        """
+        driver = cls.get_driver()
+        count = 0
+        batch_size = 200
+        # release() flips status to pending, so failed-filtered fetches naturally
+        # shrink. Cap iterations as a safety net.
+        for _ in range(1000):
+            jobs = await driver.get_jobs(
+                status="failed", queue=queue, tag=tag,
+                created_from=created_from, created_to=created_to,
+                offset=0, limit=batch_size,
+            )
+            if not jobs:
+                break
+            for job in jobs:
+                await driver.release(job, delay=0)
+                count += 1
+        return count
+
+    # ── Prune ───────────────────────────────────────────────────
+
+    @classmethod
+    async def prune(
+        cls,
+        status: str | None = None,
+        tag: str | None = None,
+        hours: float | None = None,
+        queue: str | None = None,
+    ) -> int:
+        older_than = hours * 3600 if hours else None
+        count = await cls.get_driver().prune(
+            status=status, tag=tag, older_than_seconds=older_than, queue=queue
+        )
+        cls.get_events().emit_nowait("queue.pruned", status=status, tag=tag, count=count)
+        return count
+
+    # ── Flush ───────────────────────────────────────────────────
+
+    @classmethod
+    async def flush(cls, queue: str | None = None) -> None:
+        await cls.get_driver().flush(queue)
+
+    # ── Metrics ─────────────────────────────────────────────────
+
+    @classmethod
+    async def metrics(cls, queue: str | None = None) -> dict[str, Any]:
+        return await cls.get_driver().get_metrics(queue)
+
+    # ── Reset (for testing) ─────────────────────────────────────
+
+    @classmethod
+    def reset(cls) -> None:
+        cls._driver = None
+        cls._config = None
+        cls._events = None
+
+
+def _build_payload(job: Type[Job] | Job | FunctionJob, kwargs: dict[str, Any]) -> JobPayload:
+    if isinstance(job, FunctionJob):
+        return job.build_payload(**kwargs)
+    if isinstance(job, Job):
+        return job.build_payload(**kwargs)
+    if isinstance(job, type) and issubclass(job, Job):
+        return job().build_payload(**kwargs)
+    raise TypeError(f"Expected a Job class or instance, got {type(job)}")
+
+
+def _create_driver(config: BaQueueConfig) -> BaseDriver:
+    name = config.driver.name
+    if name == "memory":
+        driver: BaseDriver = MemoryDriver()
+    elif name == "sqlite":
+        from baqueue.drivers.sqlite_driver import SqliteDriver
+        path = config.driver.url or ".baqueue.db"
+        driver = SqliteDriver(path=path, **config.driver.options)
+    elif name == "redis":
+        try:
+            from baqueue.drivers.redis_driver import RedisDriver
+        except ImportError:
+            raise ImportError(
+                "Redis driver requires the 'redis' package.\n"
+                "Install it with: pip install baqueue[redis]"
+            ) from None
+        driver = RedisDriver(config.driver.url, prefix=config.prefix, **config.driver.options)
+    elif name == "postgres":
+        try:
+            from baqueue.drivers.postgres_driver import PostgresDriver
+        except ImportError:
+            raise ImportError(
+                "PostgreSQL driver requires the 'asyncpg' package.\n"
+                "Install it with: pip install baqueue[postgres]"
+            ) from None
+        driver = PostgresDriver(config.driver.url, prefix=config.prefix, **config.driver.options)
+    else:
+        raise ValueError(f"Unknown driver: {name}")
+    driver.auto_cleanup_on_disk_full = config.auto_cleanup_on_disk_full
+    return driver

baqueue/retry.py

@@ -0,0 +1,55 @@
+"""Backoff / retry strategies for failed jobs."""
+
+from __future__ import annotations
+
+import random
+from enum import Enum
+
+
+class BackoffStrategy(str, Enum):
+    FIXED = "fixed"
+    LINEAR = "linear"
+    EXPONENTIAL = "exponential"
+
+
+def compute_delay(
+    backoff: str | list[int],
+    attempt: int,
+    base_delay: float = 5.0,
+    max_delay: float = 3600.0,
+) -> float:
+    """Compute the retry delay in seconds for a given attempt.
+
+    Args:
+        backoff: Strategy name ("fixed", "linear", "exponential") or
+                 a list of explicit delays in seconds (e.g. [10, 60, 300]).
+        attempt: Current attempt number (1-based).
+        base_delay: Base delay in seconds for computed strategies.
+        max_delay: Maximum delay cap.
+
+    Returns:
+        Delay in seconds before the next retry.
+    """
+    if isinstance(backoff, list):
+        idx = min(attempt - 1, len(backoff) - 1)
+        return float(backoff[idx])
+
+    strategy = backoff.lower()
+
+    if strategy == BackoffStrategy.FIXED:
+        return base_delay
+
+    if strategy == BackoffStrategy.LINEAR:
+        return min(base_delay * attempt, max_delay)
+
+    if strategy == BackoffStrategy.EXPONENTIAL:
+        delay = base_delay * (2 ** (attempt - 1))
+        jitter = random.uniform(0, delay * 0.1)
+        return min(delay + jitter, max_delay)
+
+    return base_delay
+
+
+def should_retry(attempt: int, max_attempts: int) -> bool:
+    """Whether the job should be retried."""
+    return attempt < max_attempts

baqueue/scheduler.py

@@ -0,0 +1,101 @@
+"""Scheduler - runs jobs on cron expressions or fixed intervals."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+
+from croniter import croniter
+
+from baqueue.config import ScheduleEntry
+from baqueue.drivers.base import BaseDriver
+from baqueue.events import EventBus
+from baqueue.serializer import JobPayload, resolve_job_class
+
+logger = logging.getLogger("baqueue.scheduler")
+
+
+class Scheduler:
+    """Runs scheduled jobs based on cron expressions or intervals.
+
+    Usage::
+
+        scheduler = Scheduler(driver, [
+            ScheduleEntry(job_class="myapp.jobs.Cleanup", cron="0 * * * *"),
+            ScheduleEntry(job_class="myapp.jobs.Heartbeat", every=30),
+        ])
+        await scheduler.start()
+    """
+
+    def __init__(
+        self,
+        driver: BaseDriver,
+        entries: list[ScheduleEntry] | None = None,
+        events: EventBus | None = None,
+    ):
+        self.driver = driver
+        self.entries = entries or []
+        self.events = events or EventBus.default()
+        self._running = False
+        self._last_run: dict[str, float] = {}
+
+    @property
+    def is_running(self) -> bool:
+        return self._running
+
+    def add(self, entry: ScheduleEntry) -> None:
+        self.entries.append(entry)
+
+    async def start(self) -> None:
+        self._running = True
+        logger.info("Scheduler started with %d entries", len(self.entries))
+
+        while self._running:
+            now = datetime.now(timezone.utc)
+            for entry in self.entries:
+                key = f"{entry.job_class}:{entry.cron or entry.every}"
+                if self._should_run(entry, now, key):
+                    await self._dispatch(entry)
+                    self._last_run[key] = now.timestamp()
+            await asyncio.sleep(1)
+
+    def stop(self) -> None:
+        self._running = False
+        logger.info("Scheduler stopped")
+
+    def _should_run(self, entry: ScheduleEntry, now: datetime, key: str) -> bool:
+        last = self._last_run.get(key, 0)
+
+        if entry.cron:
+            cron = croniter(entry.cron, datetime.fromtimestamp(last, tz=timezone.utc))
+            next_run = cron.get_next(datetime)
+            if next_run.tzinfo is None:
+                next_run = next_run.replace(tzinfo=timezone.utc)
+            return now >= next_run
+
+        if entry.every > 0:
+            return (now.timestamp() - last) >= entry.every
+
+        return False
+
+    async def _dispatch(self, entry: ScheduleEntry) -> None:
+        try:
+            cls = resolve_job_class(entry.job_class)
+            if hasattr(cls, "build_payload"):
+                if isinstance(cls, type):
+                    payload = cls().build_payload(**entry.payload)
+                else:
+                    payload = cls.build_payload(**entry.payload)
+            else:
+                payload = JobPayload(
+                    job_class=entry.job_class,
+                    data=entry.payload,
+                    queue=entry.queue,
+                )
+            payload.queue = entry.queue
+            await self.driver.push(payload)
+            logger.info("Scheduled job dispatched: %s", entry.job_class)
+            self.events.emit_nowait("schedule.dispatched", entry=entry)
+        except Exception:
+            logger.exception("Failed to dispatch scheduled job: %s", entry.job_class)