ltq 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

ltq/__init__.py

@@ -1,11 +1,17 @@
+from .app import App
+from .utils import dispatch
 from .task import Task
 from .worker import Worker
+from .scheduler import Scheduler
 from .logger import get_logger
 from .errors import RejectMessage, RetryMessage
 
 __all__ = [
+    "App",
     "Worker",
+    "Scheduler",
     "Task",
+    "dispatch",
     "get_logger",
     "RejectMessage",
     "RetryMessage",

ltq/app.py

@@ -0,0 +1,14 @@
+import asyncio
+
+from .worker import Worker
+
+
+class App:
+    def __init__(self) -> None:
+        self.workers: set[Worker] = set()
+
+    def register_worker(self, worker: Worker) -> None:
+        self.workers.add(worker)
+
+    async def run(self) -> None:
+        await asyncio.gather(*(w.run() for w in self.workers))

ltq/cli.py

@@ -7,6 +7,7 @@ from pathlib import Path
 import argparse
 
 from .logger import setup_logging, get_logger
+from .app import App
 from .worker import Worker
 
 logger = get_logger()
@@ -46,7 +47,12 @@ def main():
         epilog="Example:\n  ltq example:worker --concurrency 100",
     )
 
-    parser.add_argument("worker", help="Worker import string (module:attribute)")
+    parser.add_argument(
+        "worker", nargs="?", help="Worker import string (module:attribute)"
+    )
+    parser.add_argument(
+        "--app", dest="app", help="App import string (module:attribute)"
+    )
     parser.add_argument("--concurrency", type=int, help="Override worker concurrency")
     parser.add_argument("--poll-sleep", type=float, help="Override worker poll sleep")
     parser.add_argument(
@@ -58,27 +64,47 @@ def main():
     )
     args = parser.parse_args()
 
+    if not args.worker and not args.app:
+        parser.error("either worker or --app is required")
+    if args.worker and args.app:
+        parser.error("cannot specify both worker and --app")
+
     # Setup colored logging for CLI
     setup_logging(level=args.log_level)
-
-    worker: Worker = import_from_string(args.worker)
-
-    # Apply overrides
-    if args.concurrency:
-        worker.concurrency = args.concurrency
-    if args.poll_sleep:
-        worker.poll_sleep = args.poll_sleep
     if args.log_level:
         logger.setLevel(args.log_level)
 
-    # Print startup info
-    logger.info("Starting ltq worker")
-    logger.info("Worker: %s", args.worker)
-
-    try:
-        asyncio.run(worker.run())
-    except KeyboardInterrupt:
-        logger.info("Shutting down...")
+    if args.app:
+        app: App = import_from_string(args.app)
+
+        for w in app.workers:
+            if args.concurrency:
+                w.concurrency = args.concurrency
+            if args.poll_sleep:
+                w.poll_sleep = args.poll_sleep
+
+        logger.info("Starting ltq app")
+        logger.info("App: %s (%d workers)", args.app, len(app.workers))
+
+        try:
+            asyncio.run(app.run())
+        except KeyboardInterrupt:
+            logger.info("Shutting down...")
+    else:
+        worker: Worker = import_from_string(args.worker)
+
+        if args.concurrency:
+            worker.concurrency = args.concurrency
+        if args.poll_sleep:
+            worker.poll_sleep = args.poll_sleep
+
+        logger.info("Starting ltq worker")
+        logger.info("Worker: %s", args.worker)
+
+        try:
+            asyncio.run(worker.run())
+        except KeyboardInterrupt:
+            logger.info("Shutting down...")
 
 
 if __name__ == "__main__":

ltq/message.py

@@ -3,21 +3,25 @@ from __future__ import annotations
 import json
 import uuid
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .task import Task
 
 
 @dataclass
 class Message:
     args: tuple[Any, ...]
     kwargs: dict[str, Any]
-    task: str
+    task_name: str
+    task: Task | None = None  # only set when Message created with Task.message
     ctx: dict[str, Any] = field(default_factory=dict)
     id: str = field(default_factory=lambda: uuid.uuid4().hex)
 
     def to_json(self) -> str:
         return json.dumps(
             {
-                "task": self.task,
+                "task_name": self.task_name,
                 "id": self.id,
                 "args": self.args,
                 "kwargs": self.kwargs,

ltq/middleware.py

@@ -95,13 +95,13 @@ class Sentry(Middleware):
 
     async def handle(self, message: Message, next_handler: Handler) -> Any:
         with self.sentry.push_scope() as scope:
-            scope.set_tag("task", message.task)
+            scope.set_tag("task", message.task_name)
             scope.set_tag("message_id", message.id)
             scope.set_context(
                 "message",
                 {
                     "id": message.id,
-                    "task": message.task,
+                    "task": message.task_name,
                     "args": message.args,
                     "kwargs": message.kwargs,
                     "ctx": message.ctx,

ltq/q.py

@@ -14,12 +14,12 @@ class Queue:
         local items = {}
         for i = 1, ARGV[1] do
             local item = redis.call("RPOP", KEYS[1])
-            if item then
-                table.insert(items, item)
-            end
+            if not item then break end
+            table.insert(items, item)
         end
-        if #items > 0 then
-            redis.call("SADD", KEYS[2], unpack(items))
+        for i = 1, #items, 5000 do
+            local chunk = {unpack(items, i, math.min(i + 4999, #items))}
+            redis.call("SADD", KEYS[2], unpack(chunk))
         end
         return items
     """

ltq/scheduler.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from .message import Message
+
+try:
+    from croniter import croniter  # type: ignore[import-not-found]
+except ModuleNotFoundError as exc:
+    raise ModuleNotFoundError(
+        "Scheduler requires optional dependency 'croniter'. "
+        "Install with 'ltq[scheduler]'."
+    ) from exc
+
+from .utils import dispatch
+from .logger import get_logger
+
+if TYPE_CHECKING:
+    from .task import Task
+
+
+@dataclass
+class ScheduledJob:
+    task: Task
+    msg: Message
+    expr: str
+    _cron: croniter = field(init=False, repr=False)
+    next_run: datetime = field(init=False)
+
+    def __post_init__(self):
+        self._cron = croniter(self.expr, datetime.now())
+        self.advance()
+
+    def advance(self) -> None:
+        self.next_run = self._cron.get_next(datetime)
+
+
+class Scheduler:
+    def __init__(self, poll_interval: float = 10.0) -> None:
+        self.poll_interval = poll_interval
+        self.jobs: list[ScheduledJob] = []
+        self.logger = get_logger("ltq.scheduler")
+        self._running = False
+
+    def cron(self, expr: str, msg: Message) -> None:
+        if msg.task is None:
+            raise ValueError("Message must have a task assigned to use with scheduler")
+        self.jobs.append(ScheduledJob(msg.task, msg, expr))
+
+    def run(self) -> None:
+        self._running = True
+        self.logger.info("Starting scheduler")
+        for job in self.jobs:
+            self.logger.info(
+                f"{job.task.name} [{job.expr}] next={job.next_run:%H:%M:%S}"
+            )
+
+        loop = asyncio.new_event_loop()
+        while self._running:
+            now = datetime.now()
+            due = [job for job in self.jobs if now >= job.next_run]
+            if due:
+                try:
+                    loop.run_until_complete(dispatch([job.msg for job in due]))
+                    for job in due:
+                        self.logger.info(
+                            f"Enqueued {job.task.name} scheduled={job.next_run:%H:%M:%S}"
+                        )
+                except Exception:
+                    self.logger.exception("Failed to dispatch scheduled jobs")
+                for job in due:
+                    job.advance()
+            time.sleep(self.poll_interval)
+        loop.close()
+
+    def stop(self) -> None:
+        self._running = False

ltq/task.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from functools import update_wrapper
 from typing import Awaitable, Callable, Generic, ParamSpec, TypeVar
 
 from .message import Message
@@ -27,7 +26,8 @@ class Task(Generic[P, R]):
         return Message(
             args=args,
             kwargs=kwargs,
-            task=self.name,
+            task=self,
+            task_name=self.name,
         )
 
     async def send(self, *args: P.args, **kwargs: P.kwargs) -> str:
@@ -35,9 +35,5 @@ class Task(Generic[P, R]):
         await self.queue.put([message], ttl=self.ttl)
         return message.id
 
-    async def send_bulk(self, messages: list[Message]) -> list[str]:
-        await self.queue.put(messages, ttl=self.ttl)
-        return [message.id for message in messages]
-
     async def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R:
         return await self.fn(*args, **kwargs)

ltq/utils.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from collections import defaultdict
+
+from .message import Message
+from .q import Queue
+
+
+async def dispatch(messages: list[Message]) -> list[str]:
+    by_queue: defaultdict[Queue, list[Message]] = defaultdict(list)
+    for msg in messages:
+        if msg.task is None:
+            raise ValueError(f"Message {msg.id} has no task assigned")
+        by_queue[msg.task.queue].append(msg)
+
+    for queue, batch in by_queue.items():
+        await queue.put(batch)
+
+    return [msg.id for msg in messages]

ltq/worker.py

@@ -38,6 +38,7 @@ class Worker:
         self.concurrency: int = concurrency
         self.poll_sleep: float = poll_sleep
 
+
     def task(
         self,
         queue_name: str | None = None,
@@ -58,7 +59,7 @@ class Worker:
 
         return decorator
 
-    async def worker(self, task: Task):
+    async def processor(self, task: Task):
         async def base(message: Message) -> Any:
             return await task.fn(*message.args, **message.kwargs)
 
@@ -66,15 +67,11 @@ class Worker:
         for middleware in reversed(self.middlewares):
             handler = partial(middleware.handle, next_handler=handler)
 
-        while True:
-            messages = await task.queue.get(self.concurrency)
-            if not messages:
-                await asyncio.sleep(self.poll_sleep)
-                continue
-
-            logger.debug(f"Processing {len(messages)} messages for {task.name}")
+        sem = asyncio.Semaphore(self.concurrency)
+        pending: dict[asyncio.Task, Message] = {}
 
-            async def process(msg: Message) -> None:
+        async def process(msg: Message) -> None:
+            async with sem:
                 try:
                     await handler(msg)
                 except RetryMessage as e:
@@ -86,12 +83,26 @@ class Worker:
                         exc_info=True,
                     )
 
-            await asyncio.gather(*(process(m) for m in messages))
-            await task.queue.ack(messages)
+        while True:
+            messages = await task.queue.get(self.concurrency)
+            if not messages:
+                await asyncio.sleep(self.poll_sleep)
+                continue
+
+            logger.debug(f"Processing {len(messages)} messages for {task.name}")
+
+            for msg in messages:
+                t = asyncio.create_task(process(msg))
+                pending[t] = msg
+
+            done, _ = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+            await task.queue.ack([pending.pop(t) for t in done])
 
     async def run(self) -> None:
         try:
-            workers = (self.worker(task) for task in self.tasks)
-            await asyncio.gather(*workers)
+            processors = (self.processor(task) for task in self.tasks)
+            await asyncio.gather(*processors)
+        except asyncio.CancelledError:
+            logger.info("Worker shutting down...")
         finally:
             await self.client.aclose()

{ltq-0.1.2.dist-info → ltq-0.2.0.dist-info}/METADATA

@@ -1,12 +1,14 @@
 Metadata-Version: 2.3
 Name: ltq
-Version: 0.1.2
+Version: 0.2.0
 Summary: Add your description here
 Author: Tom Clesius
 Author-email: Tom Clesius <tomclesius@gmail.com>
 Requires-Dist: redis>=7.1.0
+Requires-Dist: croniter>=6.0.0 ; extra == 'scheduler'
 Requires-Dist: sentry-sdk>=2.0.0 ; extra == 'sentry'
 Requires-Python: >=3.13
+Provides-Extra: scheduler
 Provides-Extra: sentry
 Description-Content-Type: text/markdown
 

ltq-0.2.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+ltq/__init__.py,sha256=yZ65BhEtoQApFCOPWCL5_5b2rAIfBkw6XW5ppSpurW0,355
+ltq/app.py,sha256=PI9Zti37qpgYVzLRIbRoAMmhJ3wLdIY05bTQCkUzehA,316
+ltq/cli.py,sha256=JD_iHStkP2WuVlOz0L90Sako-oHPafTzV08INDJsHio,3219
+ltq/errors.py,sha256=i9kXtSVMpam_0VpyL5eaSfBwkGF6dK7xouAcLdc_eNc,324
+ltq/logger.py,sha256=HPClhDt3ecwZqE0Vq2oYF8Nr9jj-xrsSX9tM6enVgkA,1791
+ltq/message.py,sha256=hY_oPNMicSICV9-_o8Rma_kJO3DGJu_PFxyt1xZKMJE,871
+ltq/middleware.py,sha256=xy4VXy8uXp0vBMhdD6aSOQdzwCdbUkcuFw7FwPsqabU,3641
+ltq/q.py,sha256=3jRlPZKO4Zd5IYnb24jO_oRHtHBbG1la5SCZPI6QLI4,2537
+ltq/scheduler.py,sha256=EjXmmcyDv9Bp473Y40cTZAW6UVvkwZ_0MDE1XOjm5nI,2451
+ltq/task.py,sha256=ZKKag4u9k9hQ-lNdIWZygmGCc01DRt8a6kMQlQr_yaw,1000
+ltq/utils.py,sha256=M7EWJ-n9J3MMwofWwtxVkwgHmGeP66eoWuWDhjbx67k,536
+ltq/worker.py,sha256=gHA9xfDx1plhrC53nfO2mmapWex1JbFVNxGGSIV_ow8,3423
+ltq-0.2.0.dist-info/WHEEL,sha256=e_m4S054HL0hyR3CpOk-b7Q7fDX6BuFkgL5OjAExXas,80
+ltq-0.2.0.dist-info/entry_points.txt,sha256=OogYaOJ_RORrWtrLlEL_gTN9Vx5tkgawl8BO7G9FKcg,38
+ltq-0.2.0.dist-info/METADATA,sha256=mf5XFhk1D41Oimz_f34fiNh7quyOhXBRQu5qwEt2tnE,2396
+ltq-0.2.0.dist-info/RECORD,,

{ltq-0.1.2.dist-info → ltq-0.2.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.9.26
+Generator: uv 0.9.27
 Root-Is-Purelib: true
 Tag: py3-none-any

ltq-0.1.2.dist-info/RECORD

@@ -1,13 +0,0 @@
-ltq/__init__.py,sha256=sN88QG1vEsxFfeV7Lr381UFbLjz9oxLO04BPLY7Ae6k,229
-ltq/cli.py,sha256=sDC19BjAzUDSNY_oSsXdV5Q2HosQCi8Htq_dOmT4AXA,2370
-ltq/errors.py,sha256=i9kXtSVMpam_0VpyL5eaSfBwkGF6dK7xouAcLdc_eNc,324
-ltq/logger.py,sha256=HPClhDt3ecwZqE0Vq2oYF8Nr9jj-xrsSX9tM6enVgkA,1791
-ltq/message.py,sha256=C6gJR6KuRrnIb-l9Jna7e-XIu_aFHBhnZICUqE1_2MU,715
-ltq/middleware.py,sha256=NMRI7UiJiUTqAHOB18Gz686mQGszSu-6hJiyTDPCKdE,3631
-ltq/q.py,sha256=LddedNdb9uYD9qAjkzN0inBhvQ-mp6uJLgguEhv_TeE,2462
-ltq/task.py,sha256=JOeyulGJ-jSM_wzYyKPTPElRGx7Ncu6n1dRdn9NGwMI,1184
-ltq/worker.py,sha256=PAMW8hvTuWCUALw41TnVBmcCxXytJbgVsxALAY9qmFY,3031
-ltq-0.1.2.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
-ltq-0.1.2.dist-info/entry_points.txt,sha256=OogYaOJ_RORrWtrLlEL_gTN9Vx5tkgawl8BO7G9FKcg,38
-ltq-0.1.2.dist-info/METADATA,sha256=7Mvz05MwsRJ5ddbegQofl9iLDHgELxwY2JEsSnJGK5Q,2316
-ltq-0.1.2.dist-info/RECORD,,