saq 0.24.8__py3-none-any.whl → 0.24.10__py3-none-any.whl

saq/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Worker",
 ]
 
-__version__ = "0.24.8"
+__version__ = "0.24.10"

saq/queue/base.py

@@ -219,13 +219,18 @@ class Queue(ABC):
     def serialize(self, job: Job) -> bytes | str:
         return self._dump(job.to_dict())
 
-    def deserialize(self, payload: dict | str | bytes | None) -> Job | None:
+    def deserialize(
+        self, payload: dict | str | bytes | None, status: Status | str | None = None
+    ) -> Job | None:
         if not payload:
             return None
 
         job_dict = payload if isinstance(payload, dict) else self._load(payload)
         if job_dict.pop("queue") != self.name:
             raise ValueError(f"Job {job_dict} fetched by wrong queue: {self.name}")
+
+        if status:
+            job_dict["status"] = status
         return Job(**job_dict, queue=self)
 
     async def worker_info(

saq/queue/postgres.py

@@ -260,14 +260,14 @@ class PostgresQueue(Queue):
                 ).format(stats_table=self.stats_table),
                 {"queue": self.name},
             )
-            results = await cursor.fetchall()
+            rows = await cursor.fetchall()
         workers: dict[str, WorkerInfo] = {
             worker_id: {
                 "stats": stats,
                 "metadata": metadata,
                 "queue_key": queue_key,
             }
-            for worker_id, stats, queue_key, metadata in results
+            for worker_id, stats, queue_key, metadata in rows
         }
 
         queued = await self.count("queued")
@@ -280,7 +280,7 @@ class PostgresQueue(Queue):
                     SQL(
                         dedent(
                             """
-                            SELECT job FROM {jobs_table}
+                            SELECT job, status FROM {jobs_table}
                             WHERE status IN ('new', 'deferred', 'queued', 'active')
                               AND queue = %(queue)s
                             """
@@ -288,8 +288,8 @@ class PostgresQueue(Queue):
                     ).format(jobs_table=self.jobs_table),
                     {"queue": self.name},
                 )
-                results = await cursor.fetchall()
-            deserialized_jobs = (self.deserialize(result[0]) for result in results)
+                rows = await cursor.fetchall()
+            deserialized_jobs = (self.deserialize(*row) for row in rows)
             jobs_info = [job.to_dict() for job in deserialized_jobs if job]
         else:
             jobs_info = []
@@ -441,10 +441,10 @@ class PostgresQueue(Queue):
                     "job_lock_keyspace": self.job_lock_keyspace,
                 },
             )
-            results = await cursor.fetchall()
+            rows = await cursor.fetchall()
 
-        for key, job_bytes, objid, status in results:
-            job = self.deserialize(job_bytes)
+        for key, job_bytes, objid, status in rows:
+            job = self.deserialize(job_bytes, status)
             assert job
             if (objid or not self.job_lock_sweep) and not job.stuck:
                 continue
@@ -514,7 +514,7 @@ class PostgresQueue(Queue):
                 SQL(
                     dedent(
                         """
-                        SELECT job
+                        SELECT job, status
                         FROM {jobs_table}
                         WHERE key = %(key)s AND queue = %(queue)s
                         """
@@ -522,9 +522,9 @@ class PostgresQueue(Queue):
                 ).format(jobs_table=self.jobs_table),
                 {"key": job_key, "queue": self.name},
             )
-            job = await cursor.fetchone()
-            if job:
-                return self.deserialize(job[0])
+            row = await cursor.fetchone()
+            if row:
+                return self.deserialize(*row)
         return None
 
     async def jobs(self, job_keys: Iterable[str]) -> t.List[Job | None]:
@@ -535,7 +535,7 @@ class PostgresQueue(Queue):
                 SQL(
                     dedent(
                         """
-                        SELECT key, job
+                        SELECT key, job, status
                         FROM {jobs_table}
                         WHERE key = ANY(%(keys)s)
                         """
@@ -543,8 +543,8 @@ class PostgresQueue(Queue):
                 ).format(jobs_table=self.jobs_table),
                 {"keys": keys},
             )
-            results: dict[str, bytes | None] = dict(await cursor.fetchall())
-        return [self.deserialize(results.get(key)) for key in keys]
+            results = {r[0]: r[1:] for r in await cursor.fetchall()}
+        return [self.deserialize(*(results.get(key) or [None])) for key in keys]
 
     async def iter_jobs(
         self,
@@ -559,7 +559,7 @@ class PostgresQueue(Queue):
                     SQL(
                         dedent(
                             """
-                            SELECT key, job
+                            SELECT key, job, status
                             FROM {jobs_table}
                             WHERE
                               status = ANY(%(statuses)s)
@@ -581,9 +581,9 @@ class PostgresQueue(Queue):
                 rows = await cursor.fetchall()
 
                 if rows:
-                    for key, job_bytes in rows:
+                    for key, job_bytes, status in rows:
                         last_key = key
-                        job = self.deserialize(job_bytes)
+                        job = self.deserialize(job_bytes, status)
                         if job:
                             yield job
                 else:
@@ -638,7 +638,7 @@ class PostgresQueue(Queue):
             return
 
         async with self._dequeue_lock:
-            async with self._get_dequeue_conn() as conn, conn.cursor() as cursor:
+            async with self._get_dequeue_conn() as conn, conn.transaction(), conn.cursor() as cursor:
                 if not self._waiting:
                     return
                 await cursor.execute(
@@ -659,7 +659,6 @@ class PostgresQueue(Queue):
                                     AND queue = %(queue)s
                                     AND group_key IS NOT NULL
                                 )
-                                AND pg_try_advisory_lock({job_lock_keyspace}, lock_key)
                               ORDER BY priority, scheduled
                               LIMIT %(limit)s
                               FOR UPDATE SKIP LOCKED
@@ -667,12 +666,11 @@ class PostgresQueue(Queue):
                             UPDATE {jobs_table} SET status = 'active'
                             FROM locked_job
                             WHERE {jobs_table}.key = locked_job.key
-                            RETURNING job
+                            RETURNING job, {jobs_table}.key, status
                             """
                         )
                     ).format(
                         jobs_table=self.jobs_table,
-                        job_lock_keyspace=self.job_lock_keyspace,
                     ),
                     {
                         "queue": self.name,
@@ -682,12 +680,35 @@ class PostgresQueue(Queue):
                         "phigh": self._priorities[1],
                     },
                 )
-                results = await cursor.fetchall()
+                rows = await cursor.fetchall()
+
+                await conn.execute(
+                    SQL(
+                        dedent(
+                            """
+                            SELECT key, pg_try_advisory_lock({job_lock_keyspace}, lock_key)
+                            FROM {jobs_table}
+                            WHERE key = ANY(%(keys)s)
+                            """
+                        )
+                    ).format(
+                        jobs_table=self.jobs_table,
+                        job_lock_keyspace=self.job_lock_keyspace,
+                    ),
+                    {"keys": [key for _, key, _ in rows]},
+                )
+                lock_acquisition_results = await cursor.fetchall()
+                for key, lock_acquired in lock_acquisition_results:
+                    if not lock_acquired:
+                        logger.error(
+                            "Could not acquire lock for job %s. This may result in unexpected behavior",
+                            key,
+                        )
 
-            for result in results:
-                self._job_queue.put_nowait(self.deserialize(result[0]))
+            for job, _, status in rows:
+                self._job_queue.put_nowait(self.deserialize(job, status))
 
-            if results:
+            if rows:
                 await self._notify(DEQUEUE)
 
     async def _enqueue(self, job: Job) -> Job | None:

{saq-0.24.8.dist-info → saq-0.24.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: saq
-Version: 0.24.8
+Version: 0.24.10
 Summary: Distributed Python job queue with asyncio and redis
 Home-page: https://github.com/tobymao/saq
 Author: Toby Mao

{saq-0.24.8.dist-info → saq-0.24.10.dist-info}/RECORD

@@ -1,4 +1,4 @@
-saq/__init__.py,sha256=8Yq6PVEa09HJzvRvQV0TUGSmOPqkdsJfNGYooR-DZ10,218
+saq/__init__.py,sha256=vYIFEVix-39RzyynnCqMQVZMrhIr4ns1aeJOmEXWCt0,219
 saq/__main__.py,sha256=N4RNqnCcj7eZbM3OyYaC03_6Cot-y-SxW5Hwx6fuzKU,2440
 saq/errors.py,sha256=XPJw6J3caSAho4ZybuodIbeuGjboVabLuf3NFOEE-4Q,112
 saq/job.py,sha256=X49OuSqZeN-RVmh4AonZRrwk2wU8MeukLSxrvjGLm7E,11552
@@ -8,9 +8,9 @@ saq/types.py,sha256=GhIq2BIE_Z9hA-qS-NQXh_iPICNI0NZxOzjW0vcMgFU,3196
 saq/utils.py,sha256=NdOycT-03zxjhKM8A1i0vzKnkv1UQxvy_Zt4GnO0Zd8,1721
 saq/worker.py,sha256=W9rqK_u0bbvUbdb879pvf1f-suNnesKk1TjYqHQREc8,16817
 saq/queue/__init__.py,sha256=5LgBHGylCVvrLDcjMCcI2dRRgh0BPdz2TKOdc8NMs2E,87
-saq/queue/base.py,sha256=vRSr4ExGETlkjc9bLrtqfi_A7Zsr_IQzCMyN_b70-UM,15572
+saq/queue/base.py,sha256=wej8WBWK-S9GXYEHXTfl7Wwwm-DB3o1RB-bHMvNLfdI,15682
 saq/queue/http.py,sha256=V9S26gJbUt5AUIR2ETasSQy4Q_K30eGtguBYHpfcLGU,7739
-saq/queue/postgres.py,sha256=Zb59GWPy36K6FaW3atdYOl4KBykoFVkW8S3gwX59nAc,34719
+saq/queue/postgres.py,sha256=7FSljVrqISV8w82cv0yWvyejMpucbucsjpu8Ntoqee4,35648
 saq/queue/postgres_migrations.py,sha256=gI6j-0TzlFFSWxji3Dy9aJ-llboJBm92J4tB_YZ7qI8,2080
 saq/queue/redis.py,sha256=sa_wzUUlfPw-RZ-v_cnDEJWEFyUi3sy_3YTqG4UklOA,17754
 saq/web/__init__.py,sha256=NG9LfjgJQxNft0_iZuZ3LnX1I58SfxRwKpycjazBoGE,23
@@ -20,9 +20,9 @@ saq/web/starlette.py,sha256=i38xuNcnQvWBY3jyHHu9Uo9ILSBzOwmk5Bq06c3CQzM,4432
 saq/web/static/app.js,sha256=i6PaRvBvt96LOINBdEuKkDvVeM-GA8lJiFg4jtQ3viY,7094
 saq/web/static/pico.min.css.gz,sha256=qCxIv3wWFMQ7MkvGSHQLwxio3121VvvieOkSjw6fv6o,9263
 saq/web/static/snabbdom.js.gz,sha256=zSO3Z761TB7bYNQFFEtypD0vCuqWesqPJeE5CuV4xRg,7603
-saq-0.24.8.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
-saq-0.24.8.dist-info/METADATA,sha256=SlpZba2Kq2khnXyF5ik6kjBzcY5FqwypKbn91EbrTSM,7781
-saq-0.24.8.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-saq-0.24.8.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
-saq-0.24.8.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
-saq-0.24.8.dist-info/RECORD,,
+saq-0.24.10.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
+saq-0.24.10.dist-info/METADATA,sha256=iPnYOcZctuzl-5KOEWq7UoOBSexbYKFTtBvEM_KUr40,7782
+saq-0.24.10.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+saq-0.24.10.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
+saq-0.24.10.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
+saq-0.24.10.dist-info/RECORD,,