tasq-client-python 0.1.12__py3-none-any.whl → 0.1.16__py3-none-any.whl

tasq_client/check_type.py

@@ -12,6 +12,11 @@ class OptionalKey:
     key: str
 
 
+@dataclass
+class OptionalValue:
+    template: Any
+
+
 class CheckTypeException(Exception):
     """
     An error indicating that the type of an object does not match the expected
@@ -60,6 +65,9 @@ def check_type(template: Any, obj: Any):
             raise CheckTypeException(
                 f"expected type {template} to be float or int but got {type(obj)}"
             )
+    elif isinstance(template, OptionalValue):
+        if obj is not None:
+            _wrap_check("optional value", lambda: check_type(template.template, obj))
     else:
         if not isinstance(obj, template):
             raise CheckTypeException(f"expected type {template} but got {type(obj)}")

tasq_client/client.py

@@ -1,16 +1,17 @@
-import multiprocessing
+import random
 import sys
 import time
 import urllib.parse
 from contextlib import contextmanager
 from dataclasses import dataclass
-from multiprocessing.context import BaseContext
+from queue import Empty, Queue
+from threading import Thread
 from typing import Any, Dict, List, Optional, Tuple
 
 import requests
 from requests.adapters import HTTPAdapter, Retry
 
-from .check_type import CheckTypeException, OptionalKey, check_type
+from .check_type import CheckTypeException, OptionalKey, OptionalValue, check_type
 
 
 @dataclass
@@ -34,6 +35,8 @@ class QueueCounts:
     # server is old enough to not support rate estimation.
     rate: Optional[float] = None
 
+    modtime: Optional[int] = None
+
 
 class TasqClient:
     """
@@ -48,9 +51,10 @@ class TasqClient:
     :param username: optional username for basic authentication.
     :param password: optional password for basic authentication.
     :param max_timeout: the maximum amount of time (in seconds) to wait
-                        between attempts to pop a task in pop_running_task().
-                        Lower values mean waiting less long in the case that
-                        a new task is pushed or all tasks are finished.
+                        between attempts to pop a task in pop_running_task(),
+                        or push a task in push_blocking().
+                        Lower values mean waiting less long to pop in the case
+                        that a new task is pushed or all tasks are finished.
     :param task_timeout: if specified, override the timeout on the server with
                          a custom timeout. This can be useful if we know we
                          will be sending frequent keepalives, but the server
@@ -80,15 +84,70 @@ class TasqClient:
         self.retry_server_errors = retry_server_errors
         self.session = requests.Session()
         self._configure_session()
-        self.mp_context = multiprocessing.get_context("spawn")
 
-    def push(self, contents: str) -> str:
-        """Push a task and get its resulting ID."""
-        return self._post_form("/task/push", dict(contents=contents), type_template=str)
+    def push(self, contents: str, limit: int = 0) -> Optional[str]:
+        """
+        Push a task and get its resulting ID.
+
+        If limit is specified, then the task will not be pushed if the queue is
+        full, in which case None is returned.
+        """
+        return self._post_form(
+            f"/task/push", dict(contents=contents, limit=limit), type_template=OptionalValue(str)
+        )
+
+    def push_batch(self, ids: List[str], limit: int = 0) -> Optional[List[str]]:
+        """
+        Push a batch of tasks and get their resulting IDs.
+
+        If limit is specified, then tasks will not be pushed if the queue does
+        not have room for all the tasks at once, in which case None is
+        returned.
+
+        If limit is negative, then (-limit + batch_size) is used as the limit.
+        This effectively limits the size of the queue before a push rather than
+        after the push, to prevent large batches from being less likely to be
+        pushed than larger batches.
+        """
+        if limit < 0:
+            limit = -limit + len(ids)
+        return self._post_json(
+            f"/task/push_batch?limit={limit}", ids, type_template=OptionalValue([str])
+        )
+
+    def push_blocking(
+        self, contents: List[str], limit: int, init_wait_time: float = 1.0
+    ) -> List[str]:
+        """
+        Push one or more tasks atomically and block until they are pushed.
+
+        If the queue cannot fit the batch, this will wait to retry with random
+        exponential backoff. Backoff is randomized to mitigate starvation.
+
+        See push_batch() for details on passing a negative limit to avoid
+        starvation of larger batches when pushing from multiple processes.
 
-    def push_batch(self, ids: List[str]) -> List[str]:
-        """Push a batch of tasks and get their resulting IDs."""
-        return self._post_json("/task/push_batch", ids, type_template=[str])
+        Unlike push_batch(), the ids returned by this method will never be
+        None, since all tasks must be pushed.
+        """
+        assert isinstance(
+            contents, (list, tuple)
+        ), f"expected a list of task contents, got object of type {type(contents)}"
+        assert (
+            init_wait_time <= self.max_timeout
+        ), f"wait time {init_wait_time=} should not be larger than {self.max_timeout=}"
+        assert limit < 0 or limit >= len(contents)
+
+        cur_wait = init_wait_time
+        while True:
+            ids = self.push_batch(contents, limit=limit)
+            if ids is not None:
+                return ids
+            timeout = cur_wait * random.random()
+            time.sleep(timeout)
+            # Use summation instead of doubling to prevent really rapid
+            # growth of cur_wait with low probability.
+            cur_wait = min(cur_wait + timeout, self.max_timeout)
 
     def pop(self) -> Tuple[Optional[Task], Optional[float]]:
         """
@@ -182,9 +241,7 @@ class TasqClient:
         while True:
             task, timeout = self.pop()
             if task is not None:
-                rt = RunningTask(
-                    self, id=task.id, contents=task.contents, mp_context=self.mp_context
-                )
+                rt = RunningTask(self, id=task.id, contents=task.contents)
                 try:
                     yield rt
                     rt.completed()
@@ -200,7 +257,7 @@ class TasqClient:
     def counts(self, rate_window: int = 0) -> QueueCounts:
         """Get the number of tasks in each state within the queue."""
         data = self._get(
-            f"/counts?window={rate_window}",
+            f"/counts?window={rate_window}&includeModtime=1",
             {
                 "pending": int,
                 "running": int,
@@ -211,6 +268,14 @@ class TasqClient:
         )
         return QueueCounts(**data)
 
+    def clear(self):
+        """Deletes the queue and all tasks in it."""
+        result = self._post_form("/task/clear", dict())
+        if result is True:
+            return True
+        else:
+            raise TasqMisbehavingServerError("failed to clear queue")
+
     def __getstate__(
         self,
     ):
@@ -281,28 +346,28 @@ class RunningTask(Task):
     cancel() or completed() is called.
     """
 
-    def __init__(
-        self, client: TasqClient, *args, mp_context: Optional[BaseContext] = None, **kwargs
-    ):
+    def __init__(self, client: TasqClient, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.client = client
-        self._proc = (mp_context or multiprocessing).Process(
+        self._kill_queue = Queue()
+        self._thread = Thread(
             target=RunningTask._keepalive_worker,
             name="tasq-keepalive-worker",
             args=(
+                self._kill_queue,
                 client,
                 self.id,
             ),
             daemon=True,
         )
-        self._proc.start()
+        self._thread.start()
 
     def cancel(self):
-        if self._proc is None:
+        if self._thread is None:
             return
-        self._proc.kill()
-        self._proc.join()
-        self._proc = None
+        self._kill_queue.put(None)
+        self._thread.join()
+        self._thread = None
 
     def completed(self):
         self.cancel()
@@ -310,6 +375,7 @@ class RunningTask(Task):
 
     @staticmethod
     def _keepalive_worker(
+        kill_queue: Queue,
         client: TasqClient,
         task_id: str,
     ):
@@ -317,8 +383,19 @@ class RunningTask(Task):
             try:
                 client.keepalive(task_id)
             except Exception as exc:  # pylint: disable=broad-except
+                # Ignore the error if we killed the thread during the
+                # keepalive call.
+                try:
+                    kill_queue.get(block=False)
+                    return
+                except Empty:
+                    pass
                 print(f"exception in tasq keepalive worker: {exc}", file=sys.stderr)
-            time.sleep(client.keepalive_interval)
+            try:
+                kill_queue.get(timeout=client.keepalive_interval)
+                return
+            except Empty:
+                pass
 
 
 class TasqRemoteError(Exception):

tasq_client_python-0.1.16.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: tasq-client-python
+Version: 0.1.16
+Summary: UNKNOWN
+Home-page: UNKNOWN
+License: UNKNOWN
+Platform: UNKNOWN
+Requires-Dist: requests
+
+UNKNOWN
+

tasq_client_python-0.1.16.dist-info/RECORD

@@ -0,0 +1,8 @@
+tasq_client/__init__.py,sha256=I0ik-_c0hcVKUgx7QsE3YnoCQyAVMFKKOzoLt-jNFtE,277
+tasq_client/check_type.py,sha256=t_jreI8rf6QWS9Jf105ZvUVbwFe-uL4rMg4kZk6e4cA,2795
+tasq_client/check_type_test.py,sha256=bvhVaO-Bu18aI3J4Kxnb0H27fzDCKkTHVBWhjJMFMis,1433
+tasq_client/client.py,sha256=1f-BBYaiILfziZfzW3J4HOGwAgRbU80pbmlPX2J9wQE,15162
+tasq_client_python-0.1.16.dist-info/METADATA,sha256=Y5auv-WWFODNhED09_NvXXNcVvyq6GDUYGvlzWf-Tx8,168
+tasq_client_python-0.1.16.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
+tasq_client_python-0.1.16.dist-info/top_level.txt,sha256=JUs_FTRfs_ggMu8zusU5CSXgAl-JHhrjMXxuZay-B58,12
+tasq_client_python-0.1.16.dist-info/RECORD,,

{tasq_client_python-0.1.12.dist-info → tasq_client_python-0.1.16.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.1)
+Generator: bdist_wheel (0.37.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

tasq_client_python-0.1.12.dist-info/METADATA

@@ -1,5 +0,0 @@
-Metadata-Version: 2.1
-Name: tasq-client-python
-Version: 0.1.12
-Requires-Dist: requests
-

tasq_client_python-0.1.12.dist-info/RECORD

@@ -1,8 +0,0 @@
-tasq_client/__init__.py,sha256=I0ik-_c0hcVKUgx7QsE3YnoCQyAVMFKKOzoLt-jNFtE,277
-tasq_client/check_type.py,sha256=lEys9wV_8xMnDVbJptG5VcIAZ_Dbr85wQYYKtfaZqrk,2583
-tasq_client/check_type_test.py,sha256=bvhVaO-Bu18aI3J4Kxnb0H27fzDCKkTHVBWhjJMFMis,1433
-tasq_client/client.py,sha256=4sBHtMbAHbQQHvdkCMtxRbohDvPU95gF8-53F-Nvv3s,12245
-tasq_client_python-0.1.12.dist-info/METADATA,sha256=Rz_gV72U-eae23k_B-IhZSvG1BIDVjkzJQblYTu57Mw,88
-tasq_client_python-0.1.12.dist-info/WHEEL,sha256=5sUXSg9e4bi7lTLOHcm6QEYwO5TIF1TNbTSVFVjcJcc,92
-tasq_client_python-0.1.12.dist-info/top_level.txt,sha256=JUs_FTRfs_ggMu8zusU5CSXgAl-JHhrjMXxuZay-B58,12
-tasq_client_python-0.1.12.dist-info/RECORD,,