ltq 0.1.0__tar.gz

ltq-0.1.0/PKG-INFO

@@ -0,0 +1,109 @@
+Metadata-Version: 2.3
+Name: ltq
+Version: 0.1.0
+Summary: Add your description here
+Author: Tom Clesius
+Author-email: Tom Clesius <tomclesius@gmail.com>
+Requires-Dist: redis>=7.1.0
+Requires-Dist: sentry-sdk>=2.0.0 ; extra == 'sentry'
+Requires-Python: >=3.13
+Provides-Extra: sentry
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="assets/logo.png" alt="LTQ" width="400">
+</p>
+
+<p align="center">
+  A lightweight, Async-first task queue built on Redis.
+</p>
+
+## Installation
+
+```bash
+pip install ltq
+# or
+uv add ltq
+```
+
+## Quick Start
+
+```python
+import asyncio
+import redis.asyncio as redis
+import ltq
+
+client = redis.from_url("redis://localhost:6379")
+worker = ltq.Worker(client=client)
+
+@worker.task()
+async def send_email(to: str, subject: str, body: str) -> None:
+    # your async code here
+    pass
+
+async def main():
+    # Enqueue a task
+    await send_email.send("user@example.com", "Hello", "World")
+
+    # Or enqueue in bulk
+    messages = [
+        send_email.message("a@example.com", "Hi", "A"),
+        send_email.message("b@example.com", "Hi", "B"),
+    ]
+    await send_email.send_bulk(messages)
+
+asyncio.run(main())
+```
+
+Each task gets its own queue by default. To share a queue between tasks, pass `queue_name`:
+
+```python
+@worker.task(queue_name="emails")
+async def send_email(...): ...
+
+@worker.task(queue_name="emails")
+async def send_newsletter(...): ...
+```
+
+## Running Workers
+
+```bash
+# Run a worker
+ltq myapp:worker
+
+# With options
+ltq myapp:worker --concurrency 100 --log-level DEBUG
+```
+
+## Middleware
+
+Add middleware to handle cross-cutting concerns:
+
+```python
+from ltq.middleware import Retry, RateLimit, Timeout
+
+worker = ltq.Worker(
+    client=client,
+    middlewares=[
+        Retry(max_retries=3, min_delay=1.0),
+        RateLimit(requests_per_second=10),
+        Timeout(timeout=30.0),
+    ],
+)
+```
+
+**Built-in:** `Retry`, `RateLimit`, `Timeout`, `Sentry` (requires `ltq[sentry]`)
+
+**Custom middleware:**
+
+```python
+from ltq.middleware import Middleware, Handler
+from ltq.message import Message
+
+class Logger(Middleware):
+    async def handle(self, message: Message, next_handler: Handler):
+        print(f"Processing {message.task}")
+        result = await next_handler(message)
+        print(f"Completed {message.task}")
+        return result
+```

ltq-0.1.0/README.md

@@ -0,0 +1,97 @@
+<p align="center">
+  <img src="assets/logo.png" alt="LTQ" width="400">
+</p>
+
+<p align="center">
+  A lightweight, Async-first task queue built on Redis.
+</p>
+
+## Installation
+
+```bash
+pip install ltq
+# or
+uv add ltq
+```
+
+## Quick Start
+
+```python
+import asyncio
+import redis.asyncio as redis
+import ltq
+
+client = redis.from_url("redis://localhost:6379")
+worker = ltq.Worker(client=client)
+
+@worker.task()
+async def send_email(to: str, subject: str, body: str) -> None:
+    # your async code here
+    pass
+
+async def main():
+    # Enqueue a task
+    await send_email.send("user@example.com", "Hello", "World")
+
+    # Or enqueue in bulk
+    messages = [
+        send_email.message("a@example.com", "Hi", "A"),
+        send_email.message("b@example.com", "Hi", "B"),
+    ]
+    await send_email.send_bulk(messages)
+
+asyncio.run(main())
+```
+
+Each task gets its own queue by default. To share a queue between tasks, pass `queue_name`:
+
+```python
+@worker.task(queue_name="emails")
+async def send_email(...): ...
+
+@worker.task(queue_name="emails")
+async def send_newsletter(...): ...
+```
+
+## Running Workers
+
+```bash
+# Run a worker
+ltq myapp:worker
+
+# With options
+ltq myapp:worker --concurrency 100 --log-level DEBUG
+```
+
+## Middleware
+
+Add middleware to handle cross-cutting concerns:
+
+```python
+from ltq.middleware import Retry, RateLimit, Timeout
+
+worker = ltq.Worker(
+    client=client,
+    middlewares=[
+        Retry(max_retries=3, min_delay=1.0),
+        RateLimit(requests_per_second=10),
+        Timeout(timeout=30.0),
+    ],
+)
+```
+
+**Built-in:** `Retry`, `RateLimit`, `Timeout`, `Sentry` (requires `ltq[sentry]`)
+
+**Custom middleware:**
+
+```python
+from ltq.middleware import Middleware, Handler
+from ltq.message import Message
+
+class Logger(Middleware):
+    async def handle(self, message: Message, next_handler: Handler):
+        print(f"Processing {message.task}")
+        result = await next_handler(message)
+        print(f"Completed {message.task}")
+        return result
+```

ltq-0.1.0/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "ltq"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [{ name = "Tom Clesius", email = "tomclesius@gmail.com" }]
+requires-python = ">=3.13"
+dependencies = ["redis>=7.1.0"]
+
+[project.optional-dependencies]
+sentry = ["sentry-sdk>=2.0.0"]
+
+[project.scripts]
+ltq = "ltq.cli:main"
+
+[build-system]
+requires = ["uv_build>=0.9.26,<0.10.0"]
+build-backend = "uv_build"

ltq-0.1.0/src/ltq/__init__.py

@@ -0,0 +1,12 @@
+from .task import Task
+from .worker import Worker
+from .logger import get_logger
+from .errors import RejectMessage, RetryMessage
+
+__all__ = [
+    "Worker",
+    "Task",
+    "get_logger",
+    "RejectMessage",
+    "RetryMessage",
+]

ltq-0.1.0/src/ltq/cli.py

@@ -0,0 +1,85 @@
+"""CLI for running ltq workers."""
+
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+import argparse
+
+from .logger import setup_logging, get_logger
+from .worker import Worker
+
+logger = get_logger()
+
+
+def import_from_string(import_str: str):
+    """Import a Worker from 'module.path:worker_name'."""
+    if ":" not in import_str:
+        logger.error("Invalid format: %s", import_str)
+        logger.error("Use: module:attribute")
+        sys.exit(1)
+
+    module_str, attr_str = import_str.split(":", 1)
+
+    # Add cwd to path for local imports
+    sys.path.insert(0, str(Path.cwd()))
+
+    try:
+        module = importlib.import_module(module_str)
+        return getattr(module, attr_str)
+    except ImportError as e:
+        logger.error("Cannot import module '%s'", module_str)
+        logger.error("%s", e)
+        sys.exit(1)
+    except AttributeError:
+        logger.error("Module '%s' has no attribute '%s'", module_str, attr_str)
+        sys.exit(1)
+
+
+def main():
+    """Run a ltq worker."""
+
+    parser = argparse.ArgumentParser(
+        prog="ltq",
+        description="Run a ltq worker",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="Example:\n  ltq example:worker --concurrency 100",
+    )
+
+    parser.add_argument("worker", help="Worker import string (module:attribute)")
+    parser.add_argument("--concurrency", type=int, help="Override worker concurrency")
+    parser.add_argument("--poll-sleep", type=float, help="Override worker poll sleep")
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        default="INFO",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        help="Set logging level (default: INFO)",
+    )
+    args = parser.parse_args()
+
+    # Setup colored logging for CLI
+    setup_logging(level=args.log_level)
+
+    worker: Worker = import_from_string(args.worker)
+
+    # Apply overrides
+    if args.concurrency:
+        worker.concurrency = args.concurrency
+    if args.poll_sleep:
+        worker.poll_sleep = args.poll_sleep
+    if args.log_level:
+        logger.setLevel(args.log_level)
+
+    # Print startup info
+    logger.info("Starting ltq worker")
+    logger.info("Worker: %s", args.worker)
+
+    try:
+        asyncio.run(worker.run())
+    except KeyboardInterrupt:
+        logger.info("Shutting down...")
+
+
+if __name__ == "__main__":
+    main()

ltq-0.1.0/src/ltq/errors.py

@@ -0,0 +1,12 @@
+class RejectMessage(Exception):
+    """Signal that a message should be dropped."""
+
+    pass
+
+
+class RetryMessage(Exception):
+    """Signal that a message should be retried after a delay."""
+
+    def __init__(self, delay: float = 0.0, message: str = "") -> None:
+        self.delay = delay
+        super().__init__(message)

ltq-0.1.0/src/ltq/logger.py

@@ -0,0 +1,63 @@
+import logging
+import sys
+
+
+class ColoredFormatter(logging.Formatter):
+    """Custom formatter with colored severity levels."""
+
+    # ANSI color codes
+    RESET = "\033[0m"
+    GRAY = "\033[90m"
+    GREEN = "\033[32m"
+    BRIGHT_GREEN = "\033[92m"
+    YELLOW = "\033[33m"
+    RED = "\033[91m"
+    CYAN = "\033[36m"
+
+    COLORS = {
+        logging.DEBUG: GRAY,
+        logging.INFO: BRIGHT_GREEN,
+        logging.WARNING: YELLOW,
+        logging.ERROR: RED,
+        logging.CRITICAL: RED,
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format log record with colored severity level."""
+        color = self.COLORS.get(record.levelno, self.RESET)
+        log_time = self.formatTime(record, "%H:%M:%S")
+        timestamp = f"{self.GRAY}{log_time}{self.RESET}"
+
+        levelname = f"{color}{record.levelname:<8}{self.RESET}"
+        workername = f"{self.CYAN}{record.name:<8}{self.RESET}"
+
+        message = record.getMessage()
+        log_line = f"{timestamp} {levelname} {workername} {message}"
+
+        if record.exc_info and not record.exc_text:
+            record.exc_text = self.formatException(record.exc_info)
+
+        if record.exc_text:
+            lines = record.exc_text.split("\n")
+            log_line += "\n" + "\n".join(
+                f"  {self.GRAY}{line}{self.RESET}" for line in lines
+            )
+
+        return log_line
+
+
+def setup_logging(level: int | str = logging.INFO) -> None:
+    logger = logging.getLogger("ltq")
+
+    if not logger.handlers:
+        logger.setLevel(level)
+
+        handler = logging.StreamHandler(sys.stdout)
+        handler.setFormatter(ColoredFormatter())
+        logger.addHandler(handler)
+
+        logger.propagate = False
+
+
+def get_logger(name: str = "ltq") -> logging.Logger:
+    return logging.getLogger(name)

ltq-0.1.0/src/ltq/message.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Message:
+    args: tuple[Any, ...]
+    kwargs: dict[str, Any]
+    task: str
+    ctx: dict[str, Any] = field(default_factory=dict)
+    id: str = field(default_factory=lambda: uuid.uuid4().hex)
+
+    def to_json(self) -> str:
+        return json.dumps(
+            {
+                "task": self.task,
+                "id": self.id,
+                "args": self.args,
+                "kwargs": self.kwargs,
+                "ctx": self.ctx,
+            }
+        )
+
+    @classmethod
+    def from_json(cls, data: str | bytes) -> Message:
+        return cls(**json.loads(data))

ltq-0.1.0/src/ltq/middleware.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from abc import ABC, abstractmethod
+from typing import Any, Awaitable, Callable
+
+from .errors import RetryMessage
+from .message import Message
+from .logger import get_logger
+
+logger = get_logger()
+Handler = Callable[[Message], Awaitable[Any]]
+
+
+class Middleware(ABC):
+    @abstractmethod
+    async def handle(self, message: Message, next_handler: Handler) -> Any: ...
+
+
+class Retry(Middleware):
+    def __init__(
+        self,
+        max_retries: int = 3,
+        min_delay: float = 1.0,
+        max_delay: float = 60.0,
+        backoff: float = 2.0,
+    ):
+        self.max_retries = max_retries
+        self.min_delay = min_delay
+        self.max_delay = max_delay
+        self.backoff = backoff
+
+    async def handle(self, message: Message, next_handler: Handler) -> Any:
+        retries = message.ctx.get("retries", 0)
+
+        try:
+            return await next_handler(message)
+        except Exception as e:
+            retries += 1
+            message.ctx["retries"] = retries
+            max_retries = max(self.max_retries - 1, 0)
+
+            if retries > max_retries:
+                raise
+
+            delay = min(
+                self.min_delay * (self.backoff ** (retries - 1)),
+                self.max_delay,
+            )
+            logger.warning(
+                f"Retry attempt {retries}/{max_retries} ({type(e).__name__})",
+                exc_info=True,
+            )
+            raise RetryMessage(delay, str(e))
+
+
+class RateLimit(Middleware):
+    def __init__(self, requests_per_second: float):
+        self.min_interval = 1.0 / requests_per_second
+        self._last_request: float = 0
+        self._lock = asyncio.Lock()
+
+    async def handle(self, message: Message, next_handler: Handler) -> Any:
+        async with self._lock:
+            now = time.monotonic()
+            elapsed = now - self._last_request
+            if elapsed < self.min_interval:
+                await asyncio.sleep(self.min_interval - elapsed)
+            self._last_request = time.monotonic()
+
+        return await next_handler(message)
+
+
+class Timeout(Middleware):
+    def __init__(self, timeout: float):
+        self.timeout = timeout
+
+    async def handle(self, message: Message, next_handler: Handler) -> Any:
+        return await asyncio.wait_for(next_handler(message), timeout=self.timeout)
+
+
+class Sentry(Middleware):
+    def __init__(self, dsn: str, **kwargs: Any) -> None:
+        try:
+            import sentry_sdk  # type: ignore[import-not-found]
+        except ModuleNotFoundError as exc:
+            raise ModuleNotFoundError(
+                "Sentry middleware requires optional dependency 'sentry-sdk'. "
+                "Install with 'ltq[sentry]'."
+            ) from exc
+
+        self.sentry = sentry_sdk
+        self.sentry.init(dsn=dsn, send_default_pii=True, **kwargs)
+
+    async def handle(self, message: Message, next_handler: Handler) -> Any:
+        with self.sentry.push_scope() as scope:
+            scope.set_tag("task", message.task)
+            scope.set_tag("message_id", message.id)
+            scope.set_context(
+                "message",
+                {
+                    "id": message.id,
+                    "task": message.task,
+                    "args": message.args,
+                    "kwargs": message.kwargs,
+                    "ctx": message.ctx,
+                },
+            )
+
+            try:
+                return await next_handler(message)
+            except RetryMessage:
+                raise
+            except Exception as e:
+                self.sentry.capture_exception(e)
+                raise

ltq-0.1.0/src/ltq/q.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from .message import Message
+
+if TYPE_CHECKING:
+    from redis.asyncio import Redis
+
+
+class Queue:
+    _GET_SCRIPT = """
+        local items = {}
+        for i = 1, ARGV[1] do
+            local item = redis.call("RPOP", KEYS[1])
+            if item then
+                table.insert(items, item)
+            end
+        end
+        if #items > 0 then
+            redis.call("SADD", KEYS[2], unpack(items))
+        end
+        return items
+    """
+
+    def __init__(self, client: Redis, name: str) -> None:
+        self.client = client
+        self.name = name
+        self.queue_key = f"queue:{name}"
+        self.processing_key = f"queue:{name}:processing"
+        self._get = client.register_script(self._GET_SCRIPT)
+
+    @staticmethod
+    def _serialize(messages: list[Message]) -> list[str]:
+        return [msg.to_json() for msg in messages]
+
+    async def put(
+        self,
+        messages: list[Message],
+        delay: float = 0.0,
+        ttl: int | None = None,
+    ) -> None:
+        if not messages:
+            return
+        if delay > 0:
+            await asyncio.sleep(delay)
+        pipe = self.client.pipeline()
+        for item in self._serialize(messages):
+            pipe.lpush(self.queue_key, item)
+        if ttl:
+            pipe.expire(self.queue_key, ttl)
+        await pipe.execute()  # type: ignore
+
+    async def get(self, count: int) -> list[Message]:
+        results = await self._get(
+            keys=[self.queue_key, self.processing_key],
+            args=[count],
+        )  # type: ignore
+        return [Message.from_json(r) for r in results]
+
+    async def ack(self, messages: list[Message]) -> None:
+        if not messages:
+            return
+        items = self._serialize(messages)
+        await self.client.srem(self.processing_key, *items)  # type: ignore
+
+    async def nack(self, messages: list[Message]) -> None:
+        if not messages:
+            return
+        items = self._serialize(messages)
+        pipe = self.client.pipeline()
+        pipe.srem(self.processing_key, *items)
+        for item in items:
+            pipe.lpush(self.queue_key, item)
+        await pipe.execute()  # type: ignore
+
+    async def len(self) -> int:
+        return await self.client.llen(self.queue_key)  # type: ignore
+
+    async def clear(self) -> None:
+        await self.client.delete(self.queue_key, self.processing_key)  # type: ignore

ltq-0.1.0/src/ltq/task.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Any, Awaitable, Callable, Generic, ParamSpec
+
+from .message import Message
+from .q import Queue
+
+P = ParamSpec("P")
+
+
+class Task(Generic[P]):
+    def __init__(
+        self,
+        name: str,
+        fn: Callable[P, Awaitable[Any]],
+        queue: Queue,
+        ttl: int | None = None,
+    ) -> None:
+        self.name = name
+        self.fn = fn
+        self.queue = queue
+        self.ttl = ttl
+
+    def message(self, *args: P.args, **kwargs: P.kwargs) -> Message:
+        return Message(
+            args=args,
+            kwargs=kwargs,
+            task=self.name,
+        )
+
+    async def send(self, *args: P.args, **kwargs: P.kwargs) -> str:
+        message = self.message(*args, **kwargs)
+        await self.queue.put([message], ttl=self.ttl)
+        return message.id
+
+    async def send_bulk(self, messages: list[Message]) -> list[str]:
+        await self.queue.put(messages, ttl=self.ttl)
+        return [message.id for message in messages]

ltq-0.1.0/src/ltq/worker.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import asyncio
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, ParamSpec
+
+from .errors import RetryMessage
+from .task import Task
+from .message import Message
+from .middleware import Handler, Middleware
+from .q import Queue
+from .logger import get_logger
+
+if TYPE_CHECKING:
+    from redis.asyncio import Redis as AsyncRedis
+
+logger = get_logger()
+
+
+P = ParamSpec("P")
+
+
+class Worker:
+    def __init__(
+        self,
+        client: AsyncRedis,
+        middlewares: list[Middleware] | None = None,
+        concurrency: int = 250,
+        poll_sleep: float = 0.1,
+    ) -> None:
+        self.client: AsyncRedis = client
+        self.tasks: list[Task] = []
+        self.middlewares: list[Middleware] = middlewares or []
+        self.concurrency: int = concurrency
+        self.poll_sleep: float = poll_sleep
+
+    def task(
+        self,
+        queue_name: str | None = None,
+        ttl: int | None = None,
+    ) -> Callable[[Callable[P, Awaitable[Any]]], Task[P]]:
+        def decorator(fn: Callable[P, Awaitable[Any]]) -> Task[P]:
+            filename = Path(fn.__code__.co_filename).stem
+            task_name = f"{filename}:{fn.__qualname__}"
+            queue = Queue(self.client, queue_name or task_name)
+            task = Task(
+                name=task_name,
+                fn=fn,
+                queue=queue,
+                ttl=ttl, 
+            )
+            self.tasks.append(task)
+            return task
+
+        return decorator
+
+    async def worker(self, task: Task):
+        async def base(message: Message) -> Any:
+            return await task.fn(*message.args, **message.kwargs)
+
+        handler: Handler = base
+        for middleware in reversed(self.middlewares):
+            handler = partial(middleware.handle, next_handler=handler)
+
+        while True:
+            messages = await task.queue.get(self.concurrency)
+            if not messages:
+                await asyncio.sleep(self.poll_sleep)
+                continue
+
+            logger.debug(f"Processing {len(messages)} messages for {task.name}")
+
+            async def process(msg: Message) -> None:
+                try:
+                    await handler(msg)
+                except RetryMessage as e:
+                    logger.warning(f"Retrying in {e.delay}s: {e}")
+                    await task.queue.put([msg], delay=e.delay)
+                except Exception as e:
+                    logger.error(
+                        f"Rejected after error in {task.name}: {e}",
+                        exc_info=True,
+                    )
+
+            await asyncio.gather(*(process(m) for m in messages))
+            await task.queue.ack(messages)
+
+    async def run(self) -> None:
+        workers = (self.worker(task) for task in self.tasks)
+        await asyncio.gather(*workers)