toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/worker.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import atexit
 import base64
 import copy
 import json
@@ -22,7 +23,9 @@ import shutil
 import signal
 import socket
 import stat
+import subprocess
 import sys
+import threading
 import time
 import traceback
 from collections.abc import Iterator
@@ -59,7 +62,6 @@ class StatsDict(MagicExpando):
 
     jobs: list[MagicExpando]
 
-
 def nextChainable(
     predecessor: JobDescription, job_store: AbstractJobStore, config: Config
 ) -> Optional[JobDescription]:
@@ -165,6 +167,85 @@ def nextChainable(
     # Made it through! This job is chainable.
     return successor
 
+def unstick_worker(interval: float = 120, timeout: float = 120) -> None:
+    """
+    Thread function that tries to prevent the process from getting stuck.
+
+    Meant to be used as a daemon thread: does not have a shutdown signal but
+    cleans up on exit.
+
+    :param interval: Try to unstick the process at intervals of this many
+        seconds.
+    :param timeout: Stop child processes that take longer than this many
+        seconds to finish.
+    """
+
+    # We've observed Toil getting stuck reading the job from the job store,
+    # either due to a problem with the FileJobStore or with local temp storage,
+    # but then get unstuck as soon as someone logged in and ran lsof on the
+    # Toil process. So we make sure to do that to ourselves every once in a
+    # while as long as the worker is running.
+
+    # Figure out our process ID
+    pid = os.getpid()
+
+    child: Optional[subprocess.Popen[bytes]] = None
+
+    def clean_up_child() -> None:
+        """
+        Cleanup function to run at daemon thread shutdown when the main thread
+        terminates without shutting us down.
+
+        Also used to kill the child process if it takes too long.
+        """
+        if child is not None:
+            # Kill the child immediately if it is running
+            child.kill()
+            try:
+                # Wait one last time to try and reap the child process
+                child.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                pass
+
+    atexit.register(clean_up_child)
+
+    # TODO: If we handle daemon thread shutdown just fine, why do we bother
+    # with all the event stuff? Why not cut it?
+
+    # Wait the interval before trying the first unstick
+    time.sleep(interval)
+
+    while True:
+        # Run an lsof on our PID, which has been observed to unstick reads.
+        #
+        # We rely on the thread being able to go away and atexit() hooks
+        # happening in the middle of a wait with a timeout.
+        #
+        # We also want to handle the case where the child process gets so
+        # gummed up that it can't exit when killed.
+
+        # Preserve errors form child process but not output
+        child = subprocess.Popen(
+            ["lsof", "-p", str(pid)],
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+        )
+        try:
+            child.wait(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            logger.warning("Running lsof took too long!")
+            clean_up_child()
+            if child.returncode is None:
+                # Kill didn't take
+                logger.warning("Could not promptly kill child process: %s", child.pid)
+
+        if child.returncode != 0:
+            # Something went wrong, which is suspicious. Either it failed or it
+            # timed out and could not be killed promptly.
+            logger.warning("Could not list open files on ourselves. Return code: %s", child.returncode)
+
+        # Wait the interval.
+        time.sleep(interval)
 
 def workerScript(
     job_store: AbstractJobStore,
@@ -240,6 +321,13 @@ def workerScript(
         # We don't need to reap the child. Either it kills us, or we finish
         # before it does. Either way, init will have to clean it up for us.
 
+    ##########################################
+    # Create the worker unsticker
+    ##########################################
+    unstick_thread = threading.Thread(target=unstick_worker, args=())
+    unstick_thread.daemon = True
+    unstick_thread.start()
+
     ##########################################
     # Load the environment for the job
     ##########################################
@@ -443,13 +531,12 @@ def workerScript(
                 )
 
         ##########################################
-        # Setup the stats, if requested
+        # Setup the stats
         ##########################################
 
-        if config.stats:
-            # Remember the cores from the first job, which is how many we have reserved for us.
-            statsDict.workers.requested_cores = jobDesc.cores
-            startClock = ResourceMonitor.get_total_cpu_time()
+        # Remember the cores from the first job, which is how many we have reserved for us.
+        statsDict.workers.requested_cores = jobDesc.cores
+        startClock = ResourceMonitor.get_total_cpu_time()
 
         startTime = time.time()
         while True:
@@ -483,7 +570,7 @@ def workerScript(
                 )
                 try:
                     with job._executor(
-                        stats=statsDict if config.stats else None, fileStore=fileStore
+                        stats=statsDict, fileStore=fileStore
                     ):
                         with deferredFunctionManager.open() as defer:
                             with fileStore.open(job):
@@ -594,22 +681,21 @@ def workerScript(
         ##########################################
         # Finish up the stats
         ##########################################
-        if config.stats:
-            totalCPUTime, totalMemoryUsage = (
-                ResourceMonitor.get_total_cpu_time_and_memory_usage()
-            )
-            statsDict.workers.time = str(time.time() - startTime)
-            statsDict.workers.clock = str(totalCPUTime - startClock)
-            statsDict.workers.memory = str(totalMemoryUsage)
-            # Say the worker used the max disk we saw from any job
-            max_bytes = 0
-            for job_stats in statsDict.jobs:
-                if "disk" in job_stats:
-                    max_bytes = max(max_bytes, int(job_stats.disk))
-            statsDict.workers.disk = str(max_bytes)
-            # Count the jobs executed.
-            # TODO: toil stats could compute this but its parser is too general to hook into simply.
-            statsDict.workers.jobs_run = len(statsDict.jobs)
+        totalCPUTime, totalMemoryUsage = (
+            ResourceMonitor.get_total_cpu_time_and_memory_usage()
+        )
+        statsDict.workers.time = str(time.time() - startTime)
+        statsDict.workers.clock = str(totalCPUTime - startClock)
+        statsDict.workers.memory = str(totalMemoryUsage)
+        # Say the worker used the max disk we saw from any job
+        max_bytes = 0
+        for job_stats in statsDict.jobs:
+            if "disk" in job_stats:
+                max_bytes = max(max_bytes, int(job_stats.disk))
+        statsDict.workers.disk = str(max_bytes)
+        # Count the jobs executed.
+        # TODO: toil stats could compute this but its parser is too general to hook into simply.
+        statsDict.workers.jobs_run = len(statsDict.jobs)
 
         # log the worker log path here so that if the file is truncated the path can still be found
         if redirect_output_to_log_file:
@@ -767,16 +853,10 @@ def workerScript(
         statsDict.logs.names = [names.stats_name for names in jobDesc.get_chain()]
         statsDict.logs.messages = logMessages
 
-    if (
-        debugging
-        or config.stats
-        or statsDict.workers.logs_to_leader
-        or statsDict.workers.logging_user_streams
-    ):
-        # We have stats/logging to report back.
-        # We report even if the job attempt failed.
-        # TODO: Will that upset analysis of the stats?
-        job_store.write_logs(json.dumps(statsDict, ensure_ascii=True))
+    # We have stats/logging to report back.
+    # We report even if the job attempt failed.
+    # TODO: Will that upset analysis of the stats?
+    job_store.write_logs(json.dumps(statsDict, ensure_ascii=True))
 
     # Remove the temp dir
     cleanUp = config.cleanWorkDir

{toil-8.0.0.dist-info → toil-8.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: toil
-Version: 8.0.0
+Version: 8.2.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -37,7 +37,7 @@ Requires-Dist: requests<=2.31.0
 Requires-Dist: docker<8,>=6.1.0
 Requires-Dist: urllib3<3,>=1.26.0
 Requires-Dist: python-dateutil
-Requires-Dist: psutil<7,>=6.1.0
+Requires-Dist: psutil<8,>=6.1.0
 Requires-Dist: PyPubSub<5,>=4.0.3
 Requires-Dist: addict<2.5,>=2.2.1
 Requires-Dist: backports.zoneinfo[tzdata]; python_version < "3.9"
@@ -47,6 +47,7 @@ Requires-Dist: ruamel.yaml>=0.15
 Requires-Dist: pyyaml<7,>=6
 Requires-Dist: typing-extensions<5,>=4.6.2
 Requires-Dist: coloredlogs<16,>=15
+Requires-Dist: prompt_toolkit<4,>=3.0
 Provides-Extra: aws
 Requires-Dist: boto3-stubs[autoscaling,boto3,ec2,iam,s3,sdb,sts]<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "aws"
@@ -81,9 +82,9 @@ Requires-Dist: wdlparse==0.1.0; extra == "wdl"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
 Provides-Extra: server
 Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "server"
-Requires-Dist: flask<4,>=2.0; extra == "server"
+Requires-Dist: flask<2.3,>=2.0; extra == "server"
 Requires-Dist: werkzeug<4,>=2.0; extra == "server"
-Requires-Dist: flask-cors==5.0.0; extra == "server"
+Requires-Dist: flask-cors==5.0.1; extra == "server"
 Requires-Dist: gunicorn==23.0.0; extra == "server"
 Requires-Dist: celery<6,>=5.1.0; extra == "server"
 Requires-Dist: wes-service<5,>=4.0.0; extra == "server"
@@ -120,9 +121,9 @@ Requires-Dist: miniwdl==1.12.1; extra == "all"
 Requires-Dist: wdlparse==0.1.0; extra == "all"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
 Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "all"
-Requires-Dist: flask<4,>=2.0; extra == "all"
+Requires-Dist: flask<2.3,>=2.0; extra == "all"
 Requires-Dist: werkzeug<4,>=2.0; extra == "all"
-Requires-Dist: flask-cors==5.0.0; extra == "all"
+Requires-Dist: flask-cors==5.0.1; extra == "all"
 Requires-Dist: gunicorn==23.0.0; extra == "all"
 Requires-Dist: celery<6,>=5.1.0; extra == "all"
 Requires-Dist: wes-service<5,>=4.0.0; extra == "all"
@@ -135,11 +136,16 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary
 
+.. image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/required.json%3Fjob=cwl_badge?icon=commonwl&label=CWL%201.2%20Conformance
+   :alt: Toil CWL 1.2 Conformance Badge
+   :target: https://github.com/common-workflow-language/cwl-v1.2/blob/main/CONFORMANCE_TESTS.md
+
 .. image:: https://badges.gitter.im/bd2k-genomics-toil/Lobby.svg
    :alt: Join the chat at https://gitter.im/bd2k-genomics-toil/Lobby
    :target: https://gitter.im/bd2k-genomics-toil/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge