saq 0.24.10__py3-none-any.whl → 0.24.12__py3-none-any.whl

saq/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Worker",
 ]
 
-__version__ = "0.24.10"
+__version__ = "0.24.12"

saq/queue/base.py

@@ -28,12 +28,12 @@ if t.TYPE_CHECKING:
     from saq.types import (
         BeforeEnqueueType,
         CountKind,
-        ListenCallback,
         DumpType,
+        ListenCallback,
         LoadType,
         QueueInfo,
-        WorkerStats,
         WorkerInfo,
+        WorkerStats,
     )
 
 
@@ -370,7 +370,7 @@ class Queue(ABC):
             job_keys: sequence of job keys
             callback: callback function, if it returns truthy, break
             timeout: if timeout is truthy, wait for timeout seconds
-            poll_interval: number of seconds in between poll attempts if needed
+            poll_interval: Number of seconds between checking job status (default 0.5)
         """
 
         async def listen() -> None:
@@ -391,7 +391,13 @@ class Queue(ABC):
         else:
             await listen()
 
-    async def apply(self, job_or_func: str, timeout: float | None = None, **kwargs: t.Any) -> t.Any:
+    async def apply(
+        self,
+        job_or_func: str,
+        timeout: float | None = None,
+        poll_interval: float = 0.5,
+        **kwargs: t.Any,
+    ) -> t.Any:
         """
         Enqueue a job and wait for its result.
 
@@ -409,9 +415,12 @@ class Queue(ABC):
         Args:
             job_or_func: Same as Queue.enqueue
             timeout: If provided, how long to wait for result, else infinite (default None)
+            poll_interval: Number of seconds between checking job status (default 0.5)
             kwargs: Same as Queue.enqueue
         """
-        results = await self.map(job_or_func, timeout=timeout, iter_kwargs=[kwargs])
+        results = await self.map(
+            job_or_func, timeout=timeout, poll_interval=poll_interval, iter_kwargs=[kwargs]
+        )
         if results:
             return results[0]
         return None
@@ -450,7 +459,7 @@ class Queue(ABC):
             return_exceptions: If False (default), an exception is immediately raised as soon as any jobs
                 fail. Other jobs won't be cancelled and will continue to run.
                 If True, exceptions are treated the same as successful results and aggregated in the result list.
-            poll_interval: number of seconds in between poll attempts
+            poll_interval: Number of seconds between checking job status (default 0.5)
             kwargs: Default kwargs for all jobs. These will be overridden by those in iter_kwargs.
         """
         iter_kwargs = [

saq/queue/postgres.py

@@ -21,7 +21,7 @@ from saq.job import (
 from saq.multiplexer import Multiplexer
 from saq.queue.base import Queue, logger
 from saq.queue.postgres_migrations import get_migrations
-from saq.utils import now_seconds
+from saq.utils import now, now_seconds
 
 if t.TYPE_CHECKING:
     from collections.abc import Iterable
@@ -666,7 +666,7 @@ class PostgresQueue(Queue):
                             UPDATE {jobs_table} SET status = 'active'
                             FROM locked_job
                             WHERE {jobs_table}.key = locked_job.key
-                            RETURNING job, {jobs_table}.key, status
+                            RETURNING job, {jobs_table}.key
                             """
                         )
                     ).format(
@@ -680,7 +680,22 @@ class PostgresQueue(Queue):
                         "phigh": self._priorities[1],
                     },
                 )
-                rows = await cursor.fetchall()
+
+                jobs = []
+                keys = []
+                dequeued = now()
+
+                for payload, key in await cursor.fetchall():
+                    # there can be a race condition where a job is swept right after it is dequeued
+                    # but before it's updated for processing
+                    job = self.deserialize(payload, Status.ACTIVE)
+                    assert job
+                    job.started = dequeued
+                    job.touched = dequeued
+                    await self._update(job, status=Status.ACTIVE, connection=conn)
+
+                    jobs.append(job)
+                    keys.append(key)
 
                 await conn.execute(
                     SQL(
@@ -695,20 +710,20 @@ class PostgresQueue(Queue):
                         jobs_table=self.jobs_table,
                         job_lock_keyspace=self.job_lock_keyspace,
                     ),
-                    {"keys": [key for _, key, _ in rows]},
+                    {"keys": keys},
                 )
-                lock_acquisition_results = await cursor.fetchall()
-                for key, lock_acquired in lock_acquisition_results:
+
+                for key, lock_acquired in await cursor.fetchall():
                     if not lock_acquired:
                         logger.error(
                             "Could not acquire lock for job %s. This may result in unexpected behavior",
                             key,
                         )
 
-            for job, _, status in rows:
-                self._job_queue.put_nowait(self.deserialize(job, status))
+            if jobs:
+                for job in jobs:
+                    self._job_queue.put_nowait(job)
 
-            if rows:
                 await self._notify(DEQUEUE)
 
     async def _enqueue(self, job: Job) -> Job | None:

{saq-0.24.10.dist-info → saq-0.24.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: saq
-Version: 0.24.10
+Version: 0.24.12
 Summary: Distributed Python job queue with asyncio and redis
 Home-page: https://github.com/tobymao/saq
 Author: Toby Mao
@@ -198,6 +198,9 @@ print(job.results)
 # run a job and return the result
 print(await queue.apply("test", a=2))
 
+# run a job with custom polling interval to check status more frequently
+print(await queue.apply("test", a=2, poll_interval=0.1))
+
 # Run multiple jobs concurrently and collect the results into a list
 print(await queue.map("test", [{"a": 3}, {"a": 4}]))
 

{saq-0.24.10.dist-info → saq-0.24.12.dist-info}/RECORD

@@ -1,4 +1,4 @@
-saq/__init__.py,sha256=vYIFEVix-39RzyynnCqMQVZMrhIr4ns1aeJOmEXWCt0,219
+saq/__init__.py,sha256=-6W6CleVyyCSWJOyF4tfXRJ9nI6u-Y4KPRgZYraOhTg,219
 saq/__main__.py,sha256=N4RNqnCcj7eZbM3OyYaC03_6Cot-y-SxW5Hwx6fuzKU,2440
 saq/errors.py,sha256=XPJw6J3caSAho4ZybuodIbeuGjboVabLuf3NFOEE-4Q,112
 saq/job.py,sha256=X49OuSqZeN-RVmh4AonZRrwk2wU8MeukLSxrvjGLm7E,11552
@@ -8,9 +8,9 @@ saq/types.py,sha256=GhIq2BIE_Z9hA-qS-NQXh_iPICNI0NZxOzjW0vcMgFU,3196
 saq/utils.py,sha256=NdOycT-03zxjhKM8A1i0vzKnkv1UQxvy_Zt4GnO0Zd8,1721
 saq/worker.py,sha256=W9rqK_u0bbvUbdb879pvf1f-suNnesKk1TjYqHQREc8,16817
 saq/queue/__init__.py,sha256=5LgBHGylCVvrLDcjMCcI2dRRgh0BPdz2TKOdc8NMs2E,87
-saq/queue/base.py,sha256=wej8WBWK-S9GXYEHXTfl7Wwwm-DB3o1RB-bHMvNLfdI,15682
+saq/queue/base.py,sha256=3Fc91Cgrpp1KUTRT0nPL9lroRmPCz7WGFpZkxDyMQh4,15919
 saq/queue/http.py,sha256=V9S26gJbUt5AUIR2ETasSQy4Q_K30eGtguBYHpfcLGU,7739
-saq/queue/postgres.py,sha256=7FSljVrqISV8w82cv0yWvyejMpucbucsjpu8Ntoqee4,35648
+saq/queue/postgres.py,sha256=Yqv73KKbAO7aC7URkjLGk_XR1JWQ2AQtqoGepLNPSzw,36130
 saq/queue/postgres_migrations.py,sha256=gI6j-0TzlFFSWxji3Dy9aJ-llboJBm92J4tB_YZ7qI8,2080
 saq/queue/redis.py,sha256=sa_wzUUlfPw-RZ-v_cnDEJWEFyUi3sy_3YTqG4UklOA,17754
 saq/web/__init__.py,sha256=NG9LfjgJQxNft0_iZuZ3LnX1I58SfxRwKpycjazBoGE,23
@@ -20,9 +20,9 @@ saq/web/starlette.py,sha256=i38xuNcnQvWBY3jyHHu9Uo9ILSBzOwmk5Bq06c3CQzM,4432
 saq/web/static/app.js,sha256=i6PaRvBvt96LOINBdEuKkDvVeM-GA8lJiFg4jtQ3viY,7094
 saq/web/static/pico.min.css.gz,sha256=qCxIv3wWFMQ7MkvGSHQLwxio3121VvvieOkSjw6fv6o,9263
 saq/web/static/snabbdom.js.gz,sha256=zSO3Z761TB7bYNQFFEtypD0vCuqWesqPJeE5CuV4xRg,7603
-saq-0.24.10.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
-saq-0.24.10.dist-info/METADATA,sha256=iPnYOcZctuzl-5KOEWq7UoOBSexbYKFTtBvEM_KUr40,7782
-saq-0.24.10.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-saq-0.24.10.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
-saq-0.24.10.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
-saq-0.24.10.dist-info/RECORD,,
+saq-0.24.12.dist-info/LICENSE,sha256=p208OXrLf_dMcvuRHpcinfsJdihCqKWbqtFXpw4kyW0,1065
+saq-0.24.12.dist-info/METADATA,sha256=g_SywCzbz3nXYipojXUVz7NQRIFK55UPXVpcPrHEuvU,7913
+saq-0.24.12.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+saq-0.24.12.dist-info/entry_points.txt,sha256=HkKOud1K15_DV7AEltn8G5Ua10VqIgHaZ4BQit4fdOk,42
+saq-0.24.12.dist-info/top_level.txt,sha256=FMrrc5EiGr4sQkEDtUMHIpomnWHL9i6xT7B6lvEh8xM,4
+saq-0.24.12.dist-info/RECORD,,