tasktiger 0.19__tar.gz → 0.19.2__tar.gz

{tasktiger-0.19 → tasktiger-0.19.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tasktiger
-Version: 0.19
+Version: 0.19.2
 Summary: Python task queue
 Home-page: http://github.com/closeio/tasktiger
 License: MIT
@@ -12,9 +12,10 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 License-File: LICENSE
 Requires-Dist: click
 Requires-Dist: redis<5,>=3.3.0
@@ -40,13 +41,21 @@ TaskTiger
 Features
 --------
 
-- Per-task fork
+- Per-task fork or synchronous worker
 
-  TaskTiger forks a subprocess for each task, This comes with several benefits:
-  Memory leaks caused by tasks are avoided since the subprocess is terminated
-  when the task is finished. A hard time limit can be set for each task, after
-  which the task is killed if it hasn't completed. To ensure performance, any
-  necessary Python modules can be preloaded in the parent process.
+  By default, TaskTiger forks a subprocess for each task, This comes with
+  several benefits: Memory leaks caused by tasks are avoided since the
+  subprocess is terminated when the task is finished. A hard time limit can be
+  set for each task, after which the task is killed if it hasn't completed. To
+  ensure performance, any necessary Python modules can be preloaded in the
+  parent process.
+
+  TaskTiger also supports synchronous workers, which allows for better
+  performance due to no forking overhead, and tasks have the ability to reuse
+  network connections. To prevent memory leaks from accumulating, workers can
+  be set to shutdown after a certain amount of time, at which point a
+  supervisor can restart them. Workers also automatically exit on on hard
+  timeouts to prevent an inconsistent process state.
 
 - Unique queues
 
@@ -574,6 +583,18 @@ Workers support the following options:
 
   Store tracebacks with execution history (config defaults to ``True``).
 
+- ``--executor``
+
+  Can be ``fork`` (default) or ``sync``. Whether to execute tasks in a separate
+  process via fork, or execute them synchronously in the same proces. See
+  "Features" section for the benefits of either approach.
+
+- ``--exit-after``
+
+  Exit the worker after the time in minutes has elapsed. This is mainly useful
+  with the synchronous executor to prevent memory leaks from accumulating.
+
+
 In some cases it is convenient to have a custom TaskTiger launch script. For
 example, your application may have a ``manage.py`` command that sets up the
 environment and you may want to launch TaskTiger workers using that script. To

tasktiger-0.19/tasktiger.egg-info/PKG-INFO → tasktiger-0.19.2/README.rst

@@ -1,25 +1,3 @@
-Metadata-Version: 2.1
-Name: tasktiger
-Version: 0.19
-Summary: Python task queue
-Home-page: http://github.com/closeio/tasktiger
-License: MIT
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: MacOS :: MacOS X
-Classifier: Operating System :: POSIX
-Classifier: Operating System :: POSIX :: Linux
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
-Classifier: Programming Language :: Python :: 3.8
-License-File: LICENSE
-Requires-Dist: click
-Requires-Dist: redis<5,>=3.3.0
-Requires-Dist: structlog
-
 =========
 TaskTiger
 =========
@@ -40,13 +18,21 @@ TaskTiger
 Features
 --------
 
-- Per-task fork
+- Per-task fork or synchronous worker
+
+  By default, TaskTiger forks a subprocess for each task, This comes with
+  several benefits: Memory leaks caused by tasks are avoided since the
+  subprocess is terminated when the task is finished. A hard time limit can be
+  set for each task, after which the task is killed if it hasn't completed. To
+  ensure performance, any necessary Python modules can be preloaded in the
+  parent process.
 
-  TaskTiger forks a subprocess for each task, This comes with several benefits:
-  Memory leaks caused by tasks are avoided since the subprocess is terminated
-  when the task is finished. A hard time limit can be set for each task, after
-  which the task is killed if it hasn't completed. To ensure performance, any
-  necessary Python modules can be preloaded in the parent process.
+  TaskTiger also supports synchronous workers, which allows for better
+  performance due to no forking overhead, and tasks have the ability to reuse
+  network connections. To prevent memory leaks from accumulating, workers can
+  be set to shutdown after a certain amount of time, at which point a
+  supervisor can restart them. Workers also automatically exit on on hard
+  timeouts to prevent an inconsistent process state.
 
 - Unique queues
 
@@ -574,6 +560,18 @@ Workers support the following options:
 
   Store tracebacks with execution history (config defaults to ``True``).
 
+- ``--executor``
+
+  Can be ``fork`` (default) or ``sync``. Whether to execute tasks in a separate
+  process via fork, or execute them synchronously in the same proces. See
+  "Features" section for the benefits of either approach.
+
+- ``--exit-after``
+
+  Exit the worker after the time in minutes has elapsed. This is mainly useful
+  with the synchronous executor to prevent memory leaks from accumulating.
+
+
 In some cases it is convenient to have a custom TaskTiger launch script. For
 example, your application may have a ``manage.py`` command that sets up the
 environment and you may want to launch TaskTiger workers using that script. To

{tasktiger-0.19 → tasktiger-0.19.2}/setup.py

@@ -9,7 +9,7 @@ tests_require = install_requires + ["freezefrog", "pytest", "psutil"]
 
 setup(
     name="tasktiger",
-    version="0.19",
+    version="0.19.2",
     url="http://github.com/closeio/tasktiger",
     license="MIT",
     description="Python task queue",
@@ -26,9 +26,10 @@ setup(
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     packages=["tasktiger"],
     package_data={"tasktiger": ["lua/*.lua"]},

{tasktiger-0.19 → tasktiger-0.19.2}/tasktiger/executor.py

@@ -63,6 +63,25 @@ class Executor:
         self.connection = worker.connection
         self.config = worker.config
 
+    def heartbeat(
+        self,
+        queue: str,
+        task_ids: Collection[str],
+        log: BoundLogger,
+        locks: Collection[Lock],
+        queue_lock: Optional[Semaphore],
+    ) -> None:
+        self.worker.heartbeat(queue, task_ids)
+        for lock in locks:
+            try:
+                lock.reacquire()
+            except LockError:
+                log.warning("could not reacquire lock", lock=lock.name)
+        if queue_lock:
+            acquired, current_locks = queue_lock.renew()
+            if not acquired:
+                log.debug("queue lock renew failure")
+
     def execute(
         self,
         queue: str,
@@ -351,18 +370,7 @@ class ForkExecutor(Executor):
                     break
 
                 try:
-                    self.worker.heartbeat(queue, all_task_ids)
-                    for lock in locks:
-                        try:
-                            lock.reacquire()
-                        except LockError:
-                            log.warning(
-                                "could not reacquire lock", lock=lock.name
-                            )
-                    if queue_lock:
-                        acquired, current_locks = queue_lock.renew()
-                        if not acquired:
-                            log.debug("queue lock renew failure")
+                    self.heartbeat(queue, all_task_ids, log, locks, queue_lock)
                 except OSError as e:
                     # EINTR happens if the task completed. Since we're just
                     # renewing locks/heartbeat it's okay if we get interrupted.
@@ -386,6 +394,21 @@ class SyncExecutor(Executor):
 
     exit_worker_on_job_timeout = True
 
+    def _periodic_heartbeat(
+        self,
+        queue: str,
+        task_ids: Collection[str],
+        log: BoundLogger,
+        locks: Collection[Lock],
+        queue_lock: Optional[Semaphore],
+        stop_event: threading.Event,
+    ) -> None:
+        while not stop_event.wait(self.config["ACTIVE_TASK_UPDATE_TIMER"]):
+            try:
+                self.heartbeat(queue, task_ids, log, locks, queue_lock)
+            except Exception:
+                log.exception("task heartbeat failed")
+
     def execute(
         self,
         queue: str,
@@ -394,5 +417,27 @@ class SyncExecutor(Executor):
         locks: Collection[Lock],
         queue_lock: Optional[Semaphore],
     ) -> bool:
+        # Run heartbeat thread.
+        all_task_ids = {task.id for task in tasks}
+        stop_event = threading.Event()
+        heartbeat_thread = threading.Thread(
+            target=self._periodic_heartbeat,
+            kwargs={
+                "queue": queue,
+                "task_ids": all_task_ids,
+                "log": log,
+                "locks": locks,
+                "queue_lock": queue_lock,
+                "stop_event": stop_event,
+            },
+        )
+        heartbeat_thread.start()
+
         # Run the tasks.
-        return self.execute_tasks(tasks, log)
+        result = self.execute_tasks(tasks, log)
+
+        # Stop the heartbeat thread.
+        stop_event.set()
+        heartbeat_thread.join()
+
+        return result

{tasktiger-0.19 → tasktiger-0.19.2}/tasktiger/worker.py

@@ -674,6 +674,8 @@ class Worker:
                     lock = self.connection.lock(
                         self._key("lockv2", lock_id),
                         timeout=self.config["ACTIVE_TASK_UPDATE_TIMEOUT"],
+                        # Sync worker uses a thread to renew the lock.
+                        thread_local=False,
                     )
                     if not lock.acquire(blocking=False):
                         log.info("could not acquire lock", task_id=task.id)

tasktiger-0.19/README.rst → tasktiger-0.19.2/tasktiger.egg-info/PKG-INFO

@@ -1,3 +1,26 @@
+Metadata-Version: 2.1
+Name: tasktiger
+Version: 0.19.2
+Summary: Python task queue
+Home-page: http://github.com/closeio/tasktiger
+License: MIT
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+License-File: LICENSE
+Requires-Dist: click
+Requires-Dist: redis<5,>=3.3.0
+Requires-Dist: structlog
+
 =========
 TaskTiger
 =========
@@ -18,13 +41,21 @@ TaskTiger
 Features
 --------
 
-- Per-task fork
+- Per-task fork or synchronous worker
+
+  By default, TaskTiger forks a subprocess for each task, This comes with
+  several benefits: Memory leaks caused by tasks are avoided since the
+  subprocess is terminated when the task is finished. A hard time limit can be
+  set for each task, after which the task is killed if it hasn't completed. To
+  ensure performance, any necessary Python modules can be preloaded in the
+  parent process.
 
-  TaskTiger forks a subprocess for each task, This comes with several benefits:
-  Memory leaks caused by tasks are avoided since the subprocess is terminated
-  when the task is finished. A hard time limit can be set for each task, after
-  which the task is killed if it hasn't completed. To ensure performance, any
-  necessary Python modules can be preloaded in the parent process.
+  TaskTiger also supports synchronous workers, which allows for better
+  performance due to no forking overhead, and tasks have the ability to reuse
+  network connections. To prevent memory leaks from accumulating, workers can
+  be set to shutdown after a certain amount of time, at which point a
+  supervisor can restart them. Workers also automatically exit on on hard
+  timeouts to prevent an inconsistent process state.
 
 - Unique queues
 
@@ -552,6 +583,18 @@ Workers support the following options:
 
   Store tracebacks with execution history (config defaults to ``True``).
 
+- ``--executor``
+
+  Can be ``fork`` (default) or ``sync``. Whether to execute tasks in a separate
+  process via fork, or execute them synchronously in the same proces. See
+  "Features" section for the benefits of either approach.
+
+- ``--exit-after``
+
+  Exit the worker after the time in minutes has elapsed. This is mainly useful
+  with the synchronous executor to prevent memory leaks from accumulating.
+
+
 In some cases it is convenient to have a custom TaskTiger launch script. For
 example, your application may have a ``manage.py`` command that sets up the
 environment and you may want to launch TaskTiger workers using that script. To

{tasktiger-0.19 → tasktiger-0.19.2}/tests/test_workers.py

@@ -8,13 +8,17 @@ import pytest
 from freezefrog import FreezeTime
 
 from tasktiger import Task, Worker
+from tasktiger._internal import ACTIVE
 from tasktiger.executor import SyncExecutor
+from tasktiger.worker import LOCK_REDIS_KEY
 
+from .config import DELAY
 from .tasks import (
     exception_task,
     long_task_killed,
     long_task_ok,
     simple_task,
+    sleep_task,
     wait_for_long_task,
 )
 from .test_base import BaseTestCase
@@ -35,10 +39,12 @@ class TestMaxWorkers(BaseTestCase):
 
         # Start two workers and wait until they start processing.
         worker1 = Process(
-            target=external_worker, kwargs={"max_workers_per_queue": 2}
+            target=external_worker,
+            kwargs={"worker_kwargs": {"max_workers_per_queue": 2}},
         )
         worker2 = Process(
-            target=external_worker, kwargs={"max_workers_per_queue": 2}
+            target=external_worker,
+            kwargs={"worker_kwargs": {"max_workers_per_queue": 2}},
         )
         worker1.start()
         worker2.start()
@@ -181,3 +187,54 @@ class TestSyncExecutorWorker:
         with pytest.raises(SystemExit):
             worker.run(once=True, force_once=True)
         ensure_queues(error={"default": 1})
+
+    def test_heartbeat(self, tiger):
+        # Test both task heartbeat and lock renewal.
+        # We set unique=True so the task ID matches the lock key.
+        task = Task(tiger, sleep_task, lock=True, unique=True)
+        task.delay()
+
+        # Start a worker and wait until it starts processing.
+        worker = Process(
+            target=external_worker,
+            kwargs={
+                "patch_config": {"ACTIVE_TASK_UPDATE_TIMER": DELAY / 2},
+                "worker_kwargs": {
+                    # Test queue lock.
+                    "max_workers_per_queue": 1,
+                    "executor_class": SyncExecutor,
+                },
+            },
+        )
+        worker.start()
+
+        time.sleep(DELAY)
+
+        queue_key = tiger._key(ACTIVE, "default")
+        queue_lock_key = tiger._key(LOCK_REDIS_KEY, "default")
+        task_lock_key = tiger._key("lockv2", task.id)
+
+        conn = tiger.connection
+
+        heartbeat_1 = conn.zscore(queue_key, task.id)
+        queue_lock_1 = conn.zrange(queue_lock_key, 0, -1, withscores=True)[0][
+            1
+        ]
+        task_lock_1 = conn.pttl(task_lock_key)
+
+        time.sleep(DELAY / 2)
+
+        heartbeat_2 = conn.zscore(queue_key, task.id)
+        queue_lock_2 = conn.zrange(queue_lock_key, 0, -1, withscores=True)[0][
+            1
+        ]
+        task_lock_2 = conn.pttl(task_lock_key)
+
+        assert heartbeat_2 > heartbeat_1 > 0
+        assert queue_lock_2 > queue_lock_1 > 0
+
+        # Active task update timeout is 2 * DELAY and we renew every DELAY / 2.
+        assert task_lock_1 > DELAY
+        assert task_lock_2 > DELAY
+
+        worker.kill()

{tasktiger-0.19 → tasktiger-0.19.2}/tests/utils.py

@@ -70,7 +70,7 @@ def get_tiger():
     return tiger
 
 
-def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
+def external_worker(n=None, patch_config=None, worker_kwargs=None):
     """
     Runs a worker. To be used with multiprocessing.Pool.map.
     """
@@ -79,12 +79,11 @@ def external_worker(n=None, patch_config=None, max_workers_per_queue=None):
     if patch_config:
         tiger.config.update(patch_config)
 
-    worker = Worker(tiger)
+    if worker_kwargs is None:
+        worker_kwargs = {}
 
-    if max_workers_per_queue is not None:
-        worker.max_workers_per_queue = max_workers_per_queue
+    Worker(tiger, **worker_kwargs).run(once=True, force_once=True)
 
-    worker.run(once=True, force_once=True)
     tiger.connection.close()