saq 0.24.6__py3-none-any.whl → 0.24.9__py3-none-any.whl

saq/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Worker",
 ]
 
-__version__ = "0.24.6"
+__version__ = "0.24.9"

saq/queue/postgres.py

@@ -138,7 +138,6 @@ class PostgresQueue(Queue):
         self._waiting = 0  # Internal counter of worker tasks waiting for dequeue
         self._dequeue_conn: AsyncConnection | None = None
         self._connection_lock = asyncio.Lock()
-        self._releasing: list[str] = []
         self._has_sweep_lock = False
         self._channel = CHANNEL.format(self.name)
         self._listener = ListenMultiplexer(self.pool, self._channel)
@@ -480,30 +479,34 @@ class PostgresQueue(Queue):
 
             job.status = status or job.status
 
-            await conn.execute(
-                SQL(
-                    dedent(
-                        """
-                        UPDATE {jobs_table} SET
-                          job = %(job)s
-                          ,status = %(status)s
-                          ,scheduled = %(scheduled)s
-                          {expire_at}
-                        WHERE key = %(key)s
-                        """
-                    )
-                ).format(
-                    jobs_table=self.jobs_table,
-                    expire_at=SQL(",expire_at = %(expire_at)s" if expire_at != -1 else ""),
-                ),
-                {
-                    "job": self.serialize(job),
-                    "status": job.status,
-                    "key": job.key,
-                    "scheduled": job.scheduled,
-                    "expire_at": expire_at,
-                },
-            )
+            try:
+                await conn.execute(
+                    SQL(
+                        dedent(
+                            """
+                            UPDATE {jobs_table} SET
+                              job = %(job)s
+                              ,status = %(status)s
+                              ,scheduled = %(scheduled)s
+                              {expire_at}
+                            WHERE key = %(key)s
+                            """
+                        )
+                    ).format(
+                        jobs_table=self.jobs_table,
+                        expire_at=SQL(",expire_at = %(expire_at)s" if expire_at != -1 else ""),
+                    ),
+                    {
+                        "job": self.serialize(job),
+                        "status": job.status,
+                        "key": job.key,
+                        "scheduled": job.scheduled,
+                        "expire_at": expire_at,
+                    },
+                )
+            finally:
+                if job.status != Status.ACTIVE:
+                    await self._release_job(job.key)
 
     async def job(self, job_key: str) -> Job | None:
         async with self.pool.connection() as conn, conn.cursor() as cursor:
@@ -643,7 +646,7 @@ class PostgresQueue(Queue):
                         dedent(
                             """
                             WITH locked_job AS (
-                              SELECT key, lock_key
+                              SELECT key
                               FROM {jobs_table}
                               WHERE status = 'queued'
                                 AND queue = %(queue)s
@@ -663,13 +666,11 @@ class PostgresQueue(Queue):
                             UPDATE {jobs_table} SET status = 'active'
                             FROM locked_job
                             WHERE {jobs_table}.key = locked_job.key
-                              AND pg_try_advisory_lock({job_lock_keyspace}, locked_job.lock_key)
-                            RETURNING job
+                            RETURNING job, {jobs_table}.key
                             """
                         )
                     ).format(
                         jobs_table=self.jobs_table,
-                        job_lock_keyspace=self.job_lock_keyspace,
                     ),
                     {
                         "queue": self.name,
@@ -681,8 +682,31 @@ class PostgresQueue(Queue):
                 )
                 results = await cursor.fetchall()
 
-            for result in results:
-                self._job_queue.put_nowait(self.deserialize(result[0]))
+                await conn.execute(
+                    SQL(
+                        dedent(
+                            """
+                            SELECT key, pg_try_advisory_lock({job_lock_keyspace}, lock_key)
+                            FROM {jobs_table}
+                            WHERE key = ANY(%(keys)s)
+                            """
+                        )
+                    ).format(
+                        jobs_table=self.jobs_table,
+                        job_lock_keyspace=self.job_lock_keyspace,
+                    ),
+                    {"keys": [key for _, key in results]},
+                )
+                lock_acquisition_results = await cursor.fetchall()
+                for key, lock_acquired in lock_acquisition_results:
+                    if not lock_acquired:
+                        logger.error(
+                            "Could not acquire lock for job %s. This may result in unexpected behavior",
+                            key,
+                        )
+
+            for job, _ in results:
+                self._job_queue.put_nowait(self.deserialize(job))
 
             if results:
                 await self._notify(DEQUEUE)
@@ -839,7 +863,7 @@ class PostgresQueue(Queue):
                     ).format(jobs_table=self.jobs_table),
                     {"key": key},
                 )
-            await self._release_job(key)
+                await self._release_job(key)
 
     async def _notify(self, key: str, connection: AsyncConnection | None = None) -> None:
         async with self.nullcontext(connection) if connection else self.pool.connection() as conn:
@@ -871,26 +895,22 @@ class PostgresQueue(Queue):
         yield enter_result
 
     async def _release_job(self, key: str) -> None:
-        self._releasing.append(key)
-        if self._connection_lock.locked():
-            return
-        async with self._get_dequeue_conn() as conn:
+        async with self._get_dequeue_conn() as conn, conn:
             await conn.execute(
                 SQL(
                     dedent(
                         """
                         SELECT pg_advisory_unlock({job_lock_keyspace}, lock_key)
                         FROM {jobs_table}
-                        WHERE key = ANY(%(keys)s)
+                        WHERE key = %(key)s
                         """
                     )
                 ).format(
                     jobs_table=self.jobs_table,
                     job_lock_keyspace=self.job_lock_keyspace,
                 ),
-                {"keys": self._releasing},
+                {"key": key},
             )
-        self._releasing.clear()
 
     @cached_property
     def _job_queue(self) -> asyncio.Queue:

{saq-0.24.6.dist-info → saq-0.24.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: saq
-Version: 0.24.6
+Version: 0.24.9
 Summary: Distributed Python job queue with asyncio and redis
 Home-page: https://github.com/tobymao/saq
 Author: Toby Mao

{saq-0.24.6.dist-info → saq-0.24.9.dist-info}/RECORD

@@ -1,4 +1,4 @@
-saq/__init__.py,sha256=9-iA2pE4jP5kLkITxQjhBvPZukTSXtSbjaT22sebwnw,218
+saq/__init__.py,sha256=G_2GAPgXY3_BEk5HNQpH8pQf3RqSY6Mjx8ZIOSSTRtc,218
 saq/__main__.py,sha256=N4RNqnCcj7eZbM3OyYaC03_6Cot-y-SxW5Hwx6fuzKU,2440
 saq/errors.py,sha256=XPJw6J3caSAho4ZybuodIbeuGjboVabLuf3NFOEE-4Q,112
 saq/job.py,sha256=X49OuSqZeN-RVmh4AonZRrwk2wU8MeukLSxrvjGLm7E,11552
@@ -10,7 +10,7 @@ saq/worker.py,sha256=W9rqK_u0bbvUbdb879pvf1f-suNnesKk1TjYqHQREc8,16817
 saq/queue/__init__.py,sha256=5LgBHGylCVvrLDcjMCcI2dRRgh0BPdz2TKOdc8NMs2E,87
 saq/queue/base.py,sha256=vRSr4ExGETlkjc9bLrtqfi_A7Zsr_IQzCMyN_b70-UM,15572
 saq/queue/http.py,sha256=V9S26gJbUt5AUIR2ETasSQy4Q_K30eGtguBYHpfcLGU,7739
-saq/queue/postgres.py,sha256=PREn1HBA8E_m57cpdtcBNov1YW8658nR7iESyQ8YHhI,34682
+saq/queue/postgres.py,sha256=R3nQBNWk4rZyAm52lUQyqxl0g37OnGUlL_Ixo9UgV9o,35580
 saq/queue/postgres_migrations.py,sha256=gI6j-0TzlFFSWxji3Dy9aJ-llboJBm92J4tB_YZ7qI8,2080
 saq/queue/redis.py,sha256=sa_wzUUlfPw-RZ-v_cnDEJWEFyUi3sy_3YTqG4UklOA,17754
 saq/web/__init__.py,sha256=NG9LfjgJQxNft0_iZuZ3LnX1I58SfxRwKpycjazBoGE,23
@@ -20,9 +20,9 @@ saq/web/starlette.py,sha256=i38xuNcnQvWBY3jyHHu9Uo9ILSBzOwmk5Bq06c3CQzM,4432
 saq/web/static/app.js,sha256=i6PaRvBvt96LOINBdEuKkDvVeM-GA8lJiFg4jtQ3viY,7094
 saq/web/static/pico.min.css.gz,sha256=qCxIv3wWFMQ7MkvGSHQLwxio3121VvvieOkSjw6fv6o,9263
 saq/web/static/snabbdom.js.gz,sha256=zSO3Z761TB7bYNQFFEtypD0vCuqWesqPJeE5CuV4xRg,7603
-saq-0.24.6.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
-saq-0.24.6.dist-info/METADATA,sha256=oJf-Wl30e1v24Nri7koife_X97945RymUfb_g0bPR0I,7781
-saq-0.24.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-saq-0.24.6.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
-saq-0.24.6.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
-saq-0.24.6.dist-info/RECORD,,
+saq-0.24.9.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
+saq-0.24.9.dist-info/METADATA,sha256=zPpbi2NsgYJVaZJk2TIJm9TVB9M3IZJoinrnV-uekOQ,7781
+saq-0.24.9.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+saq-0.24.9.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
+saq-0.24.9.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
+saq-0.24.9.dist-info/RECORD,,