saq 0.24.11__py3-none-any.whl → 0.24.13__py3-none-any.whl

saq/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Worker",
 ]
 
-__version__ = "0.24.11"
+__version__ = "0.24.13"

saq/queue/postgres.py

@@ -21,7 +21,7 @@ from saq.job import (
 from saq.multiplexer import Multiplexer
 from saq.queue.base import Queue, logger
 from saq.queue.postgres_migrations import get_migrations
-from saq.utils import now_seconds
+from saq.utils import now, now_seconds
 
 if t.TYPE_CHECKING:
     from collections.abc import Iterable
@@ -666,7 +666,7 @@ class PostgresQueue(Queue):
                             UPDATE {jobs_table} SET status = 'active'
                             FROM locked_job
                             WHERE {jobs_table}.key = locked_job.key
-                            RETURNING job, {jobs_table}.key, status
+                            RETURNING job, {jobs_table}.key
                             """
                         )
                     ).format(
@@ -680,7 +680,22 @@ class PostgresQueue(Queue):
                         "phigh": self._priorities[1],
                     },
                 )
-                rows = await cursor.fetchall()
+
+                jobs = []
+                keys = []
+                dequeued = now()
+
+                for payload, key in await cursor.fetchall():
+                    # there can be a race condition where a job is swept right after it is dequeued
+                    # but before it's updated for processing
+                    job = self.deserialize(payload, Status.ACTIVE)
+                    assert job
+                    job.started = dequeued
+                    job.touched = dequeued
+                    await self._update(job, status=Status.ACTIVE, connection=conn)
+
+                    jobs.append(job)
+                    keys.append(key)
 
                 await conn.execute(
                     SQL(
@@ -695,20 +710,20 @@ class PostgresQueue(Queue):
                         jobs_table=self.jobs_table,
                         job_lock_keyspace=self.job_lock_keyspace,
                     ),
-                    {"keys": [key for _, key, _ in rows]},
+                    {"keys": keys},
                 )
-                lock_acquisition_results = await cursor.fetchall()
-                for key, lock_acquired in lock_acquisition_results:
+
+                for key, lock_acquired in await cursor.fetchall():
                     if not lock_acquired:
                         logger.error(
                             "Could not acquire lock for job %s. This may result in unexpected behavior",
                             key,
                         )
 
-            for job, _, status in rows:
-                self._job_queue.put_nowait(self.deserialize(job, status))
+            if jobs:
+                for job in jobs:
+                    self._job_queue.put_nowait(job)
 
-            if rows:
                 await self._notify(DEQUEUE)
 
     async def _enqueue(self, job: Job) -> Job | None:
@@ -873,7 +888,21 @@ class PostgresQueue(Queue):
 
     @asynccontextmanager
     async def _get_dequeue_conn(self) -> t.AsyncGenerator:
+        """
+        This context manager is used instead of the pool.connection() context manager because we need
+        to reuse specific connections outside of a context to properly manager advisory locks.
+        If advisory locks are not used, this context manager can be replaced with pool.connection().
+        """
         async with self._connection_lock:
+            # Check if the connection is still associated with a connection pool.
+            if not getattr(self._dequeue_conn, "_pool", None):
+                # The only known case of having a connection without a pool is when the connection was closed
+                # Therefore this close check shouldn't be needed but it doesn't hurt to close anyways
+                # to cover potential case where the connection is not closed and we would leave an open connection.
+                if self._dequeue_conn:
+                    await self._dequeue_conn.close()
+                self._dequeue_conn = None
+
             if self._dequeue_conn:
                 try:
                     # Pool normally performs this check when getting a connection.

{saq-0.24.11.dist-info → saq-0.24.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: saq
-Version: 0.24.11
+Version: 0.24.13
 Summary: Distributed Python job queue with asyncio and redis
 Home-page: https://github.com/tobymao/saq
 Author: Toby Mao

{saq-0.24.11.dist-info → saq-0.24.13.dist-info}/RECORD

@@ -1,4 +1,4 @@
-saq/__init__.py,sha256=su3QNUaQuTeApvIOApyA-tqOHgXQLGx-_WrSLhrDWEg,219
+saq/__init__.py,sha256=n2njj1ekrqBoS5BBfDJRAxeqL4HwEQixcjMOf12TGBc,219
 saq/__main__.py,sha256=N4RNqnCcj7eZbM3OyYaC03_6Cot-y-SxW5Hwx6fuzKU,2440
 saq/errors.py,sha256=XPJw6J3caSAho4ZybuodIbeuGjboVabLuf3NFOEE-4Q,112
 saq/job.py,sha256=X49OuSqZeN-RVmh4AonZRrwk2wU8MeukLSxrvjGLm7E,11552
@@ -10,7 +10,7 @@ saq/worker.py,sha256=W9rqK_u0bbvUbdb879pvf1f-suNnesKk1TjYqHQREc8,16817
 saq/queue/__init__.py,sha256=5LgBHGylCVvrLDcjMCcI2dRRgh0BPdz2TKOdc8NMs2E,87
 saq/queue/base.py,sha256=3Fc91Cgrpp1KUTRT0nPL9lroRmPCz7WGFpZkxDyMQh4,15919
 saq/queue/http.py,sha256=V9S26gJbUt5AUIR2ETasSQy4Q_K30eGtguBYHpfcLGU,7739
-saq/queue/postgres.py,sha256=7FSljVrqISV8w82cv0yWvyejMpucbucsjpu8Ntoqee4,35648
+saq/queue/postgres.py,sha256=HP-RKIb6OPa7I4QTrS1omWlDUPvzQRcjaB0ZrxnpYvE,37060
 saq/queue/postgres_migrations.py,sha256=gI6j-0TzlFFSWxji3Dy9aJ-llboJBm92J4tB_YZ7qI8,2080
 saq/queue/redis.py,sha256=sa_wzUUlfPw-RZ-v_cnDEJWEFyUi3sy_3YTqG4UklOA,17754
 saq/web/__init__.py,sha256=NG9LfjgJQxNft0_iZuZ3LnX1I58SfxRwKpycjazBoGE,23
@@ -20,9 +20,9 @@ saq/web/starlette.py,sha256=i38xuNcnQvWBY3jyHHu9Uo9ILSBzOwmk5Bq06c3CQzM,4432
 saq/web/static/app.js,sha256=i6PaRvBvt96LOINBdEuKkDvVeM-GA8lJiFg4jtQ3viY,7094
 saq/web/static/pico.min.css.gz,sha256=qCxIv3wWFMQ7MkvGSHQLwxio3121VvvieOkSjw6fv6o,9263
 saq/web/static/snabbdom.js.gz,sha256=zSO3Z761TB7bYNQFFEtypD0vCuqWesqPJeE5CuV4xRg,7603
-saq-0.24.11.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
-saq-0.24.11.dist-info/METADATA,sha256=IXsF53-LGEvvPVhqPFU6ZFqvH3dNFCaeySLFRCLJHME,7913
-saq-0.24.11.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-saq-0.24.11.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
-saq-0.24.11.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
-saq-0.24.11.dist-info/RECORD,,
+saq-0.24.13.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
+saq-0.24.13.dist-info/METADATA,sha256=-tfHuQvVYkcGER_GLb7mCNLAWbRkMM6wdtEa0EIMej0,7913
+saq-0.24.13.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+saq-0.24.13.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
+saq-0.24.13.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
+saq-0.24.13.dist-info/RECORD,,