saq 0.23.0__py3-none-any.whl → 0.24.3__py3-none-any.whl

saq/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Worker",
 ]
 
-__version__ = "0.23.0"
+__version__ = "0.24.3"

saq/job.py

@@ -139,12 +139,19 @@ class Job:
 
     _EXCLUDE_NON_FULL = {
         "kwargs",
+        "timeout",
+        "heartbeat",
+        "retries",
+        "ttl",
+        "retry_delay",
+        "retry_backoff",
         "scheduled",
         "progress",
-        "total_ms",
         "result",
         "error",
         "status",
+        "priority",
+        "group_key",
         "meta",
     }
 
@@ -156,28 +163,23 @@ class Job:
             full: If true, will list the full kwargs for the Job, else an abridged version.
         """
         # Using an exclusion list preserves order for kwargs below
-        excluded = set() if full else self._EXCLUDE_NON_FULL
-        kwargs = ", ".join(
-            f"{k}={v}"
-            for k, v in {
-                "function": self.function,
-                "kwargs": self.kwargs,
-                "queue": self.get_queue().name,
-                "id": self.id,
-                "scheduled": self.scheduled,
-                "progress": self.progress,
-                "process_ms": self.duration("process"),
-                "start_ms": self.duration("start"),
-                "total_ms": self.duration("total"),
-                "attempts": self.attempts,
-                "result": self.result,
-                "error": self.error,
-                "status": self.status,
-                "meta": self.meta,
-            }.items()
-            if v is not None and k not in excluded
-        )
-        return f"Job<{kwargs}>"
+
+        kwargs = {}
+
+        for field in dataclasses.fields(self):
+            key = field.name
+            value = getattr(self, key)
+            if (full or key not in self._EXCLUDE_NON_FULL) and value != field.default:
+                kwargs[key] = value
+
+        if "queue" in kwargs:
+            kwargs["queue"] = kwargs["queue"].name
+
+        if not kwargs.get("meta"):
+            kwargs.pop("meta", None)
+
+        info = ", ".join(f"{k}: {v}" for k, v in kwargs.items())
+        return f"Job<{info}>"
 
     def __repr__(self) -> str:
         return self.info(True)

saq/queue/base.py

@@ -110,7 +110,6 @@ class Queue(ABC):
     async def sweep(self, lock: int = 60, abort: float = 5.0) -> list[str]:
         pass
 
-    @abstractmethod
     async def notify(self, job: Job) -> None:
         pass
 
@@ -354,7 +353,18 @@ class Queue(ABC):
         job_keys: Iterable[str],
         callback: ListenCallback,
         timeout: float | None = 10,
+        poll_interval: float = 0.5,
     ) -> None:
+        """
+        Listen to updates on jobs.
+
+        Args:
+            job_keys: sequence of job keys
+            callback: callback function, if it returns truthy, break
+            timeout: if timeout is truthy, wait for timeout seconds
+            poll_interval: number of seconds in between poll attempts if needed
+        """
+
         async def listen() -> None:
             while True:
                 for job in await self.jobs(job_keys):
@@ -366,7 +376,7 @@ class Queue(ABC):
                         stop = callback(job.id, job.status)
                     if stop:
                         return
-                await asyncio.sleep(1)
+                await asyncio.sleep(poll_interval)
 
         if timeout:
             await asyncio.wait_for(listen(), timeout)
@@ -404,6 +414,7 @@ class Queue(ABC):
         iter_kwargs: Sequence[dict[str, t.Any]],
         timeout: float | None = None,
         return_exceptions: bool = False,
+        poll_interval: float = 0.5,
         **kwargs: t.Any,
     ) -> list[t.Any]:
         """
@@ -431,6 +442,7 @@ class Queue(ABC):
             return_exceptions: If False (default), an exception is immediately raised as soon as any jobs
                 fail. Other jobs won't be cancelled and will continue to run.
                 If True, exceptions are treated the same as successful results and aggregated in the result list.
+            poll_interval: number of seconds in between poll attempts
             kwargs: Default kwargs for all jobs. These will be overridden by those in iter_kwargs.
         """
         iter_kwargs = [
@@ -442,46 +454,24 @@ class Queue(ABC):
             }
             for kw in iter_kwargs
         ]
-        job_keys = [key["key"] for key in iter_kwargs]
-        pending_job_keys = set(job_keys)
-
-        def callback(job_key: str, status: Status) -> bool:
-            if status in TERMINAL_STATUSES:
-                pending_job_keys.discard(job_key)
-
-            if status in UNSUCCESSFUL_TERMINAL_STATUSES and not return_exceptions:
-                return True
-
-            if not pending_job_keys:
-                return True
 
-            return False
-
-        # Start listening before we enqueue the jobs.
-        # This ensures we don't miss any updates.
-        task = asyncio.create_task(self.listen(pending_job_keys, callback, timeout=None))
-
-        try:
-            await asyncio.gather(*(self.enqueue(job_or_func, **kw) for kw in iter_kwargs))
-        except Exception:
-            task.cancel()
-            raise
-
-        await asyncio.wait_for(task, timeout=timeout)
-
-        results = []
-
-        for job in await self.jobs(job_keys):
-            if job is None:
-                continue
-            if job.status in UNSUCCESSFUL_TERMINAL_STATUSES:
-                exc = JobError(job)
-                if not return_exceptions:
-                    raise exc
-                results.append(exc)
-            else:
-                results.append(job.result)
-        return results
+        await asyncio.gather(*(self.enqueue(job_or_func, **kw) for kw in iter_kwargs))
+        incomplete = object()
+        results = {key["key"]: incomplete for key in iter_kwargs}
+
+        while remaining := [k for k, v in results.items() if v is incomplete]:
+            for key, job in zip(remaining, await self.jobs(remaining)):
+                if not job:
+                    results[key] = None
+                elif job.status in UNSUCCESSFUL_TERMINAL_STATUSES:
+                    exc = JobError(job)
+                    if not return_exceptions:
+                        raise exc
+                    results[key] = exc
+                elif job.status in TERMINAL_STATUSES:
+                    results[key] = job.result
+            await asyncio.sleep(poll_interval)
+        return list(results.values())
 
     @asynccontextmanager
     async def batch(self) -> AsyncIterator[None]:

saq/queue/postgres.py

@@ -28,7 +28,6 @@ if t.TYPE_CHECKING:
 
     from saq.types import (
         CountKind,
-        ListenCallback,
         DumpType,
         LoadType,
         QueueInfo,
@@ -70,9 +69,6 @@ class PostgresQueue(Queue):
         max_size: maximum pool size. (default 20)
         If greater than 0, this limits the maximum number of connections to Postgres.
             Otherwise, maintain `min_size` number of connections.
-        poll_interval: how often to poll for jobs. (default 1)
-            If 0, the queue will not poll for jobs and will only rely on notifications from the server.
-            This mean cron jobs will not be picked up in a timely fashion.
         saq_lock_keyspace: The first of two advisory lock keys used by SAQ. (default 0)
             SAQ uses advisory locks for coordinating tasks between its workers, e.g. sweeping.
         job_lock_keyspace: The first of two advisory lock keys used for jobs. (default 1)
@@ -103,7 +99,6 @@ class PostgresQueue(Queue):
         load: LoadType | None = None,
         min_size: int = 4,
         max_size: int = 20,
-        poll_interval: int = 1,
         saq_lock_keyspace: int = 0,
         job_lock_keyspace: int = 1,
         job_lock_sweep: bool = True,
@@ -125,10 +120,12 @@ class PostgresQueue(Queue):
             check=AsyncConnectionPool.check_connection,
             open=False,
         )
+        if self.pool.kwargs.get("autocommit") is False:
+            raise ValueError("SAQ Connection pool must have autocommit enabled.")
+        self.pool.kwargs["autocommit"] = True
         self._is_pool_provided = pool is not None
         self.min_size = min_size
         self.max_size = max_size
-        self.poll_interval = poll_interval
         self.saq_lock_keyspace = saq_lock_keyspace
         self.job_lock_keyspace = job_lock_keyspace
         self.job_lock_sweep = job_lock_sweep
@@ -145,7 +142,7 @@ class PostgresQueue(Queue):
         self._connected = False
 
     async def init_db(self) -> None:
-        async with self.pool.connection() as conn, conn.cursor() as cursor, conn.transaction():
+        async with self._get_dequeue_conn() as conn, conn.cursor() as cursor, conn.transaction():
             await cursor.execute(
                 SQL("SELECT pg_try_advisory_lock(%(key1)s, 0)"),
                 {"key1": self.saq_lock_keyspace},
@@ -231,9 +228,12 @@ class PostgresQueue(Queue):
     async def disconnect(self) -> None:
         if not self._connected:
             return
+
         async with self._connection_lock:
             if self._dequeue_conn:
                 await self._dequeue_conn.cancel_safe()
+                async with self._dequeue_conn as conn:
+                    await conn.execute("SELECT pg_advisory_unlock_all()")
                 await self.pool.putconn(self._dequeue_conn)
                 self._dequeue_conn = None
         if not self._is_pool_provided:
@@ -359,7 +359,7 @@ class PostgresQueue(Queue):
 
         if not self._has_sweep_lock:
             # Attempt to get the sweep lock and hold on to it
-            async with self._get_dequeue_conn() as conn, conn.cursor() as cursor, conn.transaction():
+            async with self._get_dequeue_conn() as conn, conn.cursor() as cursor:
                 await cursor.execute(
                     SQL("SELECT pg_try_advisory_lock(%(key1)s, hashtext(%(queue)s))"),
                     {
@@ -446,6 +446,10 @@ class PostgresQueue(Queue):
                 continue
 
             swept.append(key)
+            logger.info(
+                "Sweeping %s, objid %s", job.info(logger.isEnabledFor(logging.DEBUG)), objid
+            )
+
             await self.abort(job, error=self.swept_error_message)
 
             try:
@@ -453,35 +457,12 @@ class PostgresQueue(Queue):
             except asyncio.TimeoutError:
                 logger.info("Could not abort job %s", key)
 
-            logger.info("Sweeping job %s", job.info(logger.isEnabledFor(logging.DEBUG)))
             if job.retryable:
                 await self.retry(job, error=self.swept_error_message)
             else:
                 await self.finish(job, Status.ABORTED, error=self.swept_error_message)
         return swept
 
-    async def listen(
-        self,
-        job_keys: Iterable[str],
-        callback: ListenCallback,
-        timeout: float | None = 10,
-    ) -> None:
-        if not job_keys:
-            return
-
-        async for message in self._listener.listen(*job_keys, timeout=timeout):
-            job_key = message["key"]
-            status = Status[message["data"].upper()]
-            if asyncio.iscoroutinefunction(callback):
-                stop = await callback(job_key, status)
-            else:
-                stop = callback(job_key, status)
-            if stop:
-                break
-
-    async def notify(self, job: Job, connection: AsyncConnection | None = None) -> None:
-        await self._notify(job.key, job.status, connection)
-
     async def _update(self, job: Job, status: Status | None = None, **kwargs: t.Any) -> None:
         expire_at = kwargs.pop("expire_at", -1)
         connection = kwargs.pop("connection", None)
@@ -518,7 +499,6 @@ class PostgresQueue(Queue):
                     "expire_at": expire_at,
                 },
             )
-            await self.notify(job, conn)
 
     async def job(self, job_key: str) -> Job | None:
         async with self.pool.connection() as conn, conn.cursor() as cursor:
@@ -629,7 +609,7 @@ class PostgresQueue(Queue):
             else:
                 async with self._listen_lock:
                     async for payload in self._listener.listen(ENQUEUE, DEQUEUE, timeout=timeout):
-                        if payload["key"] == ENQUEUE:
+                        if payload == ENQUEUE:
                             await self._dequeue()
 
                         if not self._job_queue.empty():
@@ -650,7 +630,7 @@ class PostgresQueue(Queue):
             return
 
         async with self._dequeue_lock:
-            async with self._get_dequeue_conn() as conn, conn.cursor() as cursor, conn.transaction():
+            async with self._get_dequeue_conn() as conn, conn.cursor() as cursor:
                 if not self._waiting:
                     return
                 await cursor.execute(
@@ -838,14 +818,12 @@ class PostgresQueue(Queue):
     ) -> None:
         key = job.key
 
-        async with self.nullcontext(
-            connection
-        ) if connection else self.pool.connection() as conn, conn.cursor() as cursor:
+        async with self.nullcontext(connection) if connection else self.pool.connection() as conn:
             if job.ttl >= 0:
                 expire_at = now_seconds() + job.ttl if job.ttl > 0 else None
                 await self.update(job, status=status, expire_at=expire_at, connection=conn)
             else:
-                await cursor.execute(
+                await conn.execute(
                     SQL(
                         dedent(
                             """
@@ -856,22 +834,12 @@ class PostgresQueue(Queue):
                     ).format(jobs_table=self.jobs_table),
                     {"key": key},
                 )
-                await self.notify(job, conn)
             await self._release_job(key)
 
-    async def _notify(
-        self, key: str, data: t.Any | None = None, connection: AsyncConnection | None = None
-    ) -> None:
-        payload = {"key": key}
-
-        if data is not None:
-            payload["data"] = data
-
+    async def _notify(self, key: str, connection: AsyncConnection | None = None) -> None:
         async with self.nullcontext(connection) if connection else self.pool.connection() as conn:
             await conn.execute(
-                SQL("NOTIFY {channel}, {payload}").format(
-                    channel=Identifier(self._channel), payload=json.dumps(payload)
-                )
+                SQL("NOTIFY {channel}, {key}").format(channel=Identifier(self._channel), key=key)
             )
 
     @asynccontextmanager
@@ -917,7 +885,6 @@ class PostgresQueue(Queue):
                 ),
                 {"keys": self._releasing},
             )
-            await conn.commit()
         self._releasing.clear()
 
     @cached_property
@@ -934,8 +901,6 @@ class ListenMultiplexer(Multiplexer):
     async def _start(self) -> None:
         async with self.pool.connection() as conn:
             await conn.execute(SQL("LISTEN {}").format(Identifier(self.key)))
-            await conn.commit()
 
             async for notify in conn.notifies():
-                payload = json.loads(notify.payload)
-                self.publish(payload["key"], payload)
+                self.publish(notify.payload, notify.payload)

saq/queue/redis.py

@@ -365,15 +365,8 @@ class RedisQueue(Queue):
         job_keys: Iterable[str],
         callback: ListenCallback,
         timeout: float | None = 10,
+        poll_interval: float = 0.5,
     ) -> None:
-        """
-        Listen to updates on jobs.
-
-        Args:
-            job_keys: sequence of job keys
-            callback: callback function, if it returns truthy, break
-            timeout: if timeout is truthy, wait for timeout seconds
-        """
         job_ids = [self.job_id(job_key) for job_key in job_keys]
 
         if not job_ids:

saq/utils.py

@@ -64,8 +64,12 @@ def exponential_backoff(
     return backoff
 
 
-async def cancel_tasks(tasks: Iterable[asyncio.Task]) -> None:
+async def cancel_tasks(tasks: Iterable[asyncio.Task], timeout: float = 1.0) -> None:
     """Cancel tasks and wait for all of them to finish"""
     for task in tasks:
         task.cancel()
-    await asyncio.gather(*tasks, return_exceptions=True)
+
+    try:
+        await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout)
+    except (asyncio.TimeoutError, asyncio.CancelledError):
+        pass

saq/worker.py

@@ -283,7 +283,8 @@ class Worker:
 
             if task and not task.done():
                 task_data["aborted"] = "abort" if job.error is None else job.error
-                await cancel_tasks([task])
+                # abort should be a blocking operation
+                await cancel_tasks([task], 0)
 
             await self.queue.finish_abort(job)
 
@@ -317,12 +318,12 @@ class Worker:
                 else:
                     await job.retry("cancelled")
         except Exception as ex:
-            logger.exception("Error processing job %s", job)
-
             if context is not None:
                 context["exception"] = ex
 
             if job:
+                logger.exception("Error processing job %s", job)
+
                 error = traceback.format_exc()
 
                 if job.retryable:

{saq-0.23.0.dist-info → saq-0.24.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: saq
-Version: 0.23.0
+Version: 0.24.3
 Summary: Distributed Python job queue with asyncio and redis
 Home-page: https://github.com/tobymao/saq
 Author: Toby Mao

{saq-0.23.0.dist-info → saq-0.24.3.dist-info}/RECORD

@@ -1,18 +1,18 @@
-saq/__init__.py,sha256=P423_WovoYFfOmEd7hg_jjEZDpnvlNEYdbL7hS1uqQw,218
+saq/__init__.py,sha256=Qn50Cd572uzUjWhCPkgGI9F7rrYXBOlP6LOZuSCJk3Y,218
 saq/__main__.py,sha256=N4RNqnCcj7eZbM3OyYaC03_6Cot-y-SxW5Hwx6fuzKU,2440
 saq/errors.py,sha256=XPJw6J3caSAho4ZybuodIbeuGjboVabLuf3NFOEE-4Q,112
-saq/job.py,sha256=Hmr4g_RfMBSOtw9E4lOyEE6wVqaXvDxXf3j1Mipq-xo,11528
+saq/job.py,sha256=Pion_buhc4N-5mqnqfwfpzVjv-paP3HHqtMAKB6XIcE,11327
 saq/multiplexer.py,sha256=S_mjo7kObSBQ_f8epf0pT5Tjxg-LABW3fSH4dPfZxsE,2332
 saq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 saq/types.py,sha256=GhIq2BIE_Z9hA-qS-NQXh_iPICNI0NZxOzjW0vcMgFU,3196
-saq/utils.py,sha256=S5QPutQibm435vsR8TTO9ehTsRvKsnsWxUg6kVkc4oE,1586
-saq/worker.py,sha256=mQofGvJ7Rig29MEtI2OVyT0rUHdteQZonVKQq8pnwV8,16668
+saq/utils.py,sha256=NdOycT-03zxjhKM8A1i0vzKnkv1UQxvy_Zt4GnO0Zd8,1721
+saq/worker.py,sha256=wd8LTfF7YQRRrOrxqvZcOpEF4KISD9Ws5WDXm3BFfnY,16730
 saq/queue/__init__.py,sha256=5LgBHGylCVvrLDcjMCcI2dRRgh0BPdz2TKOdc8NMs2E,87
-saq/queue/base.py,sha256=eKpCme_PwTTuhSOHE9YmlVbNwkBwOVRGpD4VyD61KQc,15467
+saq/queue/base.py,sha256=KaRv9r0fmlRf2CH6Q72MGHXIk9e3vpPEWo4vt5DL1RA,15460
 saq/queue/http.py,sha256=V9S26gJbUt5AUIR2ETasSQy4Q_K30eGtguBYHpfcLGU,7739
-saq/queue/postgres.py,sha256=8stUQefR29l8zJ1rEKv9-TwOGdy7XVzj9GufeA0B5gY,35519
+saq/queue/postgres.py,sha256=UKnARw9at2b6gV3cynZ0jR46Hiir50o40dEd7gEClE4,34307
 saq/queue/postgres_migrations.py,sha256=gI6j-0TzlFFSWxji3Dy9aJ-llboJBm92J4tB_YZ7qI8,2080
-saq/queue/redis.py,sha256=ODtvqny5LmUWh-HqccV4GiR6BFUmWPD2PM7CzUv1dAw,17972
+saq/queue/redis.py,sha256=sa_wzUUlfPw-RZ-v_cnDEJWEFyUi3sy_3YTqG4UklOA,17754
 saq/web/__init__.py,sha256=NG9LfjgJQxNft0_iZuZ3LnX1I58SfxRwKpycjazBoGE,23
 saq/web/aiohttp.py,sha256=JklrRWt0aPSVkxRXbWC9l278lfsOSNo1TFjBOjsl03w,3925
 saq/web/common.py,sha256=U-TALY06werM4gIeGqW-V3HAu8Tko1EP0uA_4wUFmHY,997
@@ -20,9 +20,9 @@ saq/web/starlette.py,sha256=i38xuNcnQvWBY3jyHHu9Uo9ILSBzOwmk5Bq06c3CQzM,4432
 saq/web/static/app.js,sha256=i6PaRvBvt96LOINBdEuKkDvVeM-GA8lJiFg4jtQ3viY,7094
 saq/web/static/pico.min.css.gz,sha256=qCxIv3wWFMQ7MkvGSHQLwxio3121VvvieOkSjw6fv6o,9263
 saq/web/static/snabbdom.js.gz,sha256=zSO3Z761TB7bYNQFFEtypD0vCuqWesqPJeE5CuV4xRg,7603
-saq-0.23.0.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
-saq-0.23.0.dist-info/METADATA,sha256=mpfZ6BqjIgO-4zQR8I0WQTrlXoPg1ISCTNOcqPjZJzA,7472
-saq-0.23.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-saq-0.23.0.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
-saq-0.23.0.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
-saq-0.23.0.dist-info/RECORD,,
+saq-0.24.3.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
+saq-0.24.3.dist-info/METADATA,sha256=gZztgr-2voE3tpOPdufquUPrSV-Y2RFmbXynMcpFHWQ,7472
+saq-0.24.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+saq-0.24.3.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
+saq-0.24.3.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
+saq-0.24.3.dist-info/RECORD,,