toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/server/utils.py

@@ -193,7 +193,7 @@ class AbstractStateStore:
     This is a key-value store, with keys namespaced by workflow ID. Concurrent
     access from multiple threads or processes is safe and globally consistent.
 
-    Keys and workflow IDs are restricted to [-a-zA-Z0-9_], because backends may
+    Keys and workflow IDs are restricted to ``[-a-zA-Z0-9_]``, because backends may
     use them as path or URL components.
 
     Key values are either a string, or None if the key is not set.

toil/server/wes/abstract_backend.py

@@ -3,7 +3,6 @@ import functools
 import json
 import logging
 import os
-import tempfile
 from abc import abstractmethod
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 from urllib.parse import urldefrag
@@ -11,6 +10,8 @@ from urllib.parse import urldefrag
 import connexion  # type: ignore
 from werkzeug.utils import secure_filename
 
+from toil.lib.io import mkdtemp
+
 logger = logging.getLogger(__name__)
 
 # Define a type for WES task log entries in responses
@@ -210,7 +211,7 @@ class WESBackend:
                          If None, a temporary directory is created.
         """
         if not temp_dir:
-            temp_dir = tempfile.mkdtemp()
+            temp_dir = mkdtemp()
         body: Dict[str, Any] = {}
         has_attachments = False
         for key, ls in connexion.request.files.lists():

toil/server/wes/amazon_wes_utils.py

@@ -20,11 +20,10 @@
 
 import json
 import logging
-import os
 import sys
 import zipfile
 from os import path
-from typing import IO, Any, Dict, List, Optional, Union, cast
+from typing import IO, List, Optional, cast
 
 if sys.version_info >= (3, 8):
     from typing import TypedDict
@@ -164,15 +163,17 @@ def parse_workflow_manifest_file(manifest_file: str) -> WorkflowPlan:
     :rtype: dict of `data` and `files`
 
     MANIFEST.json is expected to be formatted like:
+    
     .. code-block:: json
+        
        {
            "mainWorkflowURL": "relpath/to/workflow",
            "inputFileURLs": [
                "relpath/to/input-file-1",
                "relpath/to/input-file-2",
-               ...
+               "relpath/to/input-file-3"
            ],
-           "optionsFileURL" "relpath/to/option-file
+           "optionsFileURL": "relpath/to/option-file"
        }
 
     The `mainWorkflowURL` property that provides a relative file path in the zip to a workflow file, which will be set as `workflowSource`

toil/server/wes/tasks.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import fcntl
 import json
 import logging
 import multiprocessing
@@ -134,8 +133,8 @@ class ToilWorkflowRunner:
         the workflow execution engine.
 
         :param workflow_engine_parameters: User-specified parameters for this
-        particular workflow. Keys are command-line options, and values are
-        option arguments, or None for options that are flags.
+            particular workflow. Keys are command-line options, and values are
+            option arguments, or None for options that are flags.
         """
         options = []
 

toil/server/wes/toil_backend.py

@@ -16,9 +16,8 @@ import logging
 import os
 import shutil
 import uuid
-from collections import Counter, defaultdict
+from collections import Counter
 from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
 from typing import (Any,
                     Callable,
                     Dict,
@@ -32,18 +31,11 @@ from typing import (Any,
                     overload)
 
 from flask import send_from_directory
-from flask.globals import request as flask_request
 from werkzeug.utils import redirect
 from werkzeug.wrappers.response import Response
 
 import toil.server.wes.amazon_wes_utils as amazon_wes_utils
-from toil.bus import (JobAnnotationMessage,
-                      JobCompletedMessage,
-                      JobFailedMessage,
-                      JobIssuedMessage,
-                      JobUpdatedMessage,
-                      MessageBus,
-                      replay_message_bus, JobStatus)
+from toil.bus import JobStatus, replay_message_bus
 from toil.lib.io import AtomicFileCreate
 from toil.lib.threading import global_mutex
 from toil.server.utils import (WorkflowStateMachine,

toil/server/wsgi_app.py

@@ -21,6 +21,8 @@ class GunicornApplication(BaseApplication):  # type: ignore
     An entry point to integrate a Gunicorn WSGI server in Python. To start a
     WSGI application with callable `app`, run the following code:
 
+    .. code-block:: python
+
         WSGIApplication(app, options={
             ...
         }).run()

toil/serviceManager.py

@@ -122,7 +122,8 @@ class ServiceManager:
         try:
             client_id = self.__clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -139,7 +140,8 @@ class ServiceManager:
         try:
             client_id = self.__failed_clients_out.get(timeout=maxWait)
             self.__waiting_clients.remove(client_id)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return client_id
         except Empty:
@@ -154,7 +156,8 @@ class ServiceManager:
         """
         try:
             service_id = self.__services_out.get(timeout=maxWait)
-            assert self.__service_manager_jobs >= 0
+            if self.__service_manager_jobs < 0:
+                raise RuntimeError("The number of jobs scheduled by the service manager cannot be negative.")
             self.__service_manager_jobs -= 1
             return service_id
         except Empty:
@@ -304,7 +307,8 @@ class ServiceManager:
                         starting_services.remove(service_id)
                         client_id = service_to_client[service_id]
                         remaining_services_by_client[client_id] -= 1
-                        assert remaining_services_by_client[client_id] >= 0
+                        if remaining_services_by_client[client_id] < 0:
+                            raise RuntimeError("The number of remaining services cannot be negative.")
                         del service_to_client[service_id]
                         if not self.__job_store.file_exists(service_job_desc.errorJobStoreID):
                             logger.error(
@@ -356,12 +360,10 @@ class ServiceManager:
                     service_job_desc,
                     service_job_desc.startJobStoreID,
                 )
-                assert self.__job_store.file_exists(
-                    service_job_desc.startJobStoreID
-                ), f"Service manager attempted to start service {service_job_desc} that has already started"
-                assert self.__toil_state.job_exists(
-                    str(service_job_desc.jobStoreID)
-                ), f"Service manager attempted to start service {service_job_desc} that is not in the job store"
+                if not self.__job_store.file_exists(service_job_desc.startJobStoreID):
+                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that has already started")
+                if not self.__toil_state.job_exists(str(service_job_desc.jobStoreID)):
+                    raise RuntimeError(f"Service manager attempted to start service {service_job_desc} that is not in the job store")
                 # At this point the terminateJobStoreID and errorJobStoreID
                 # could have been deleted, since the service can be killed at
                 # any time! So we can't assert their presence here.

toil/statsAndLogging.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import gzip
+import io
 import json
 import logging
 import os
@@ -49,7 +50,7 @@ class StatsAndLogging:
         self._worker.start()
 
     @classmethod
-    def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], job_name: Optional[str] = None) -> str:
+    def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], stream_name: str) -> str:
         """
         Given a stream of text or bytes, and the job name, job itself, or some
         other optional stringifyable identity info for the job, return a big
@@ -62,7 +63,7 @@ class StatsAndLogging:
 
         :param stream: The stream of text or bytes to print for the user.
         """
-        lines = [f'Log from job "{job_name}" follows:', '=========>']
+        lines = [f'{stream_name} follows:', '=========>']
 
         for line in stream:
             if isinstance(line, bytes):
@@ -75,13 +76,13 @@ class StatsAndLogging:
 
 
     @classmethod
-    def logWithFormatting(cls, jobStoreID: str, jobLogs: Union[IO[str], IO[bytes]], method: Callable[[str], None] = logger.debug,
+    def logWithFormatting(cls, stream_name: str, jobLogs: Union[IO[str], IO[bytes]], method: Callable[[str], None] = logger.debug,
                             message: Optional[str] = None) -> None:
         if message is not None:
             method(message)
 
-        # Format and log the logs, identifying the job with its job store ID.
-        method(cls.formatLogStream(jobLogs, jobStoreID))
+        # Format and log the logs, identifying the stream with the given name.
+        method(cls.formatLogStream(jobLogs, stream_name))
 
     @classmethod
     def writeLogFiles(cls, jobNames: List[str], jobLogList: List[str], config: 'Config', failed: bool = False) -> None:
@@ -95,7 +96,7 @@ class StatsAndLogging:
             logName = ('failed_' if failed else '') + logName
             counter = 0
             while True:
-                suffix = str(counter).zfill(3) + logExtension
+                suffix = '_' + str(counter).zfill(3) + logExtension
                 fullName = os.path.join(logPath, logName + suffix)
                 #  The maximum file name size in the default HFS+ file system is 255 UTF-16 encoding units, so basically 255 characters
                 if len(fullName) >= 255:
@@ -118,6 +119,9 @@ class StatsAndLogging:
             # we don't have anywhere to write the logs, return now
             return
 
+        # Make sure the destination exists
+        os.makedirs(path, exist_ok=True)
+
         fullName = createName(path, mainFileName, extension, failed)
         with writeFn(fullName, 'wb') as f:
             for l in jobLogList:
@@ -148,16 +152,42 @@ class StatsAndLogging:
             if not isinstance(statsStr, str):
                 statsStr = statsStr.decode()
             stats = json.loads(statsStr, object_hook=Expando)
+            if not stats:
+                return
+
             try:
-                logs = stats.workers.logsToMaster
+                # Handle all the log_to_leader messages
+                logs = stats.workers.logs_to_leader
             except AttributeError:
-                # To be expected if there were no calls to logToMaster()
+                # To be expected if there were no calls to log_to_leader()
                 pass
             else:
                 for message in logs:
                     logger.log(int(message.level),
                                'Got message from job at time %s: %s',
                                time.strftime('%m-%d-%Y %H:%M:%S'), message.text)
+
+            try:
+                # Handle all the user-level text streams reported back (command output, etc.)
+                user_logs = stats.workers.logging_user_streams
+            except AttributeError:
+                # To be expected if there were no calls to log_user_stream()
+                pass
+            else:
+                for stream_entry in user_logs:
+                    try:
+                        # Unpack the stream name and text.
+                        name, text = stream_entry.name, stream_entry.text
+                    except AttributeError:
+                        # Doesn't have a user-provided stream name and stream
+                        # text, so skip it.
+                        continue
+                    # Since this is sent as inline text we need to pretend to stream it.
+                    # TODO: Save these as individual files if they start to get too big?
+                    cls.logWithFormatting(name, io.StringIO(text), logger.info)
+                    # Save it as a log file, as if it were a Toil-level job.
+                    cls.writeLogFiles([name], [text], config=config)
+
             try:
                 logs = stats.logs
             except AttributeError:
@@ -166,7 +196,7 @@ class StatsAndLogging:
                 # we may have multiple jobs per worker
                 jobNames = logs.names
                 messages = logs.messages
-                cls.logWithFormatting(jobNames[0], messages,
+                cls.logWithFormatting(f'Log from job "{jobNames[0]}"', messages,
                                       message='Received Toil worker log. Disable debug level logging to hide this output')
                 cls.writeLogFiles(jobNames, messages, config=config)
 
@@ -225,6 +255,8 @@ def add_logging_options(parser: ArgumentParser) -> None:
     levels += [l.lower() for l in levels] + [l.upper() for l in levels]
     group.add_argument("--logOff", dest="logLevel", default=default_loglevel,
                        action="store_const", const="CRITICAL", help="Same as --logCRITICAL.")
+    # Maybe deprecate the above in favor of --logLevel?
+
     group.add_argument("--logLevel", dest="logLevel", default=default_loglevel, choices=levels,
                        help=f"Set the log level. Default: {default_loglevel}.  Options: {levels}.")
     group.add_argument("--logFile", dest="logFile", help="File to log in.")

toil/test/__init__.py

@@ -21,7 +21,6 @@ import shutil
 import signal
 import subprocess
 import sys
-import tempfile
 import threading
 import time
 import unittest
@@ -30,6 +29,7 @@ from abc import ABCMeta, abstractmethod
 from contextlib import contextmanager
 from inspect import getsource
 from shutil import which
+from tempfile import mkstemp
 from textwrap import dedent
 from typing import (Any,
                     Callable,
@@ -57,6 +57,7 @@ from toil import ApplianceImageNotFound, applianceSelf, toilPackageDirPath
 from toil.lib.accelerators import (have_working_nvidia_docker_runtime,
                                    have_working_nvidia_smi)
 from toil.lib.aws import running_on_ec2
+from toil.lib.io import mkdtemp
 from toil.lib.iterables import concat
 from toil.lib.memoize import memoize
 from toil.lib.threading import ExceptionalThread, cpu_count
@@ -188,7 +189,7 @@ class ToilTest(unittest.TestCase):
         prefix.extend([_f for _f in names if _f])
         prefix.append('')
         temp_dir_path = os.path.realpath(
-            tempfile.mkdtemp(dir=cls._tempBaseDir, prefix="-".join(prefix))
+            mkdtemp(dir=cls._tempBaseDir, prefix="-".join(prefix))
         )
         cls._tempDirs.append(temp_dir_path)
         return temp_dir_path
@@ -314,7 +315,7 @@ else:
 def get_temp_file(suffix: str = "", rootDir: Optional[str] = None) -> str:
     """Return a string representing a temporary file, that must be manually deleted."""
     if rootDir is None:
-        handle, tmp_file = tempfile.mkstemp(suffix)
+        handle, tmp_file = mkstemp(suffix)
         os.close(handle)
         return tmp_file
     else:
@@ -359,10 +360,17 @@ def needs_rsync3(test_item: MT) -> MT:
     return test_item
 
 
+def needs_online(test_item: MT) -> MT:
+    """Use as a decorator before test classes or methods to run only if we are meant to talk to the Internet."""
+    test_item = _mark_test('online', test_item)
+    if os.getenv('TOIL_SKIP_ONLINE', '').lower() == 'true':
+        return unittest.skip('Skipping online test.')(test_item)
+    return test_item
+
 def needs_aws_s3(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if AWS S3 is usable."""
     # TODO: we just check for generic access to the AWS account
-    test_item = _mark_test('aws-s3', test_item)
+    test_item = _mark_test('aws-s3', needs_online(test_item))
     try:
         from boto import config
         boto_credentials = config.get('Credentials', 'aws_access_key_id')
@@ -415,7 +423,7 @@ def needs_google_storage(test_item: MT) -> MT:
     Cloud is installed and we ought to be able to access public Google Storage
     URIs.
     """
-    test_item = _mark_test('google-storage', test_item)
+    test_item = _mark_test('google-storage', needs_online(test_item))
     try:
         from google.cloud import storage  # noqa
     except ImportError:
@@ -427,7 +435,7 @@ def needs_google_project(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if we have a Google Cloud project set.
     """
-    test_item = _mark_test('google-project', test_item)
+    test_item = _mark_test('google-project', needs_online(test_item))
     test_item = needs_env_var('TOIL_GOOGLE_PROJECTID', "a Google project ID")(test_item)
     return test_item
 
@@ -447,44 +455,19 @@ def needs_torque(test_item: MT) -> MT:
         return test_item
     return unittest.skip("Install PBS/Torque to include this test.")(test_item)
 
-
-def needs_tes(test_item: MT) -> MT:
-    """Use as a decorator before test classes or methods to run only if TES is available."""
-    test_item = _mark_test('tes', test_item)
-
-    try:
-        from toil.batchSystems.tes import TESBatchSystem
-    except ImportError:
-        return unittest.skip("Install py-tes to include this test")(test_item)
-
-    tes_url = os.environ.get('TOIL_TES_ENDPOINT', TESBatchSystem.get_default_tes_endpoint())
-    try:
-        urlopen(tes_url)
-    except HTTPError:
-        # Funnel happens to 404 if TES is working. But any HTTPError means we
-        # dialed somebody who picked up.
-        pass
-    except URLError:
-        # Will give connection refused if we can't connect because the server's
-        # not there. We can also get a "cannot assign requested address" if
-        # we're on Kubernetes dialing localhost and !!creative things!! have
-        # been done to the network stack.
-        return unittest.skip(f"Run a TES server on {tes_url} to include this test")(test_item)
-    return test_item
-
-
 def needs_kubernetes_installed(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Kubernetes is installed."""
     test_item = _mark_test('kubernetes', test_item)
     try:
         import kubernetes
+        str(kubernetes)  # to prevent removal of this import
     except ImportError:
         return unittest.skip("Install Toil with the 'kubernetes' extra to include this test.")(test_item)
     return test_item
 
 def needs_kubernetes(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Kubernetes is installed and configured."""
-    test_item = needs_kubernetes_installed(test_item)
+    test_item = needs_kubernetes_installed(needs_online(test_item))
     try:
         import kubernetes
         try:
@@ -514,14 +497,6 @@ def needs_mesos(test_item: MT) -> MT:
     return test_item
 
 
-def needs_parasol(test_item: MT) -> MT:
-    """Use as decorator so tests are only run if Parasol is installed."""
-    test_item = _mark_test('parasol', test_item)
-    if which('parasol'):
-        return test_item
-    return unittest.skip("Install Parasol to include this test.")(test_item)
-
-
 def needs_slurm(test_item: MT) -> MT:
     """Use as a decorator before test classes or methods to run only if Slurm is installed."""
     test_item = _mark_test('slurm', test_item)
@@ -571,20 +546,20 @@ def needs_docker(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to only run them if
     docker is installed and docker-based tests are enabled.
     """
-    test_item = _mark_test('docker', test_item)
+    test_item = _mark_test('docker', needs_online(test_item))
     if os.getenv('TOIL_SKIP_DOCKER', '').lower() == 'true':
         return unittest.skip('Skipping docker test.')(test_item)
     if which('docker'):
         return test_item
     else:
         return unittest.skip("Install docker to include this test.")(test_item)
-        
+
 def needs_singularity(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to only run them if
     singularity is installed.
     """
-    test_item = _mark_test('singularity', test_item)
+    test_item = _mark_test('singularity', needs_online(test_item))
     if which('singularity'):
         return test_item
     else:
@@ -621,7 +596,7 @@ def needs_docker_cuda(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to only run them if
     a CUDA setup is available through Docker.
     """
-    test_item = _mark_test('docker_cuda', test_item)
+    test_item = _mark_test('docker_cuda', needs_online(test_item))
     if have_working_nvidia_docker_runtime():
         return test_item
     else:
@@ -677,7 +652,7 @@ def needs_celery_broker(test_item: MT) -> MT:
     """
     Use as a decorator before test classes or methods to run only if RabbitMQ is set up to take Celery jobs.
     """
-    test_item = _mark_test('celery', test_item)
+    test_item = _mark_test('celery', needs_online(test_item))
     test_item = needs_env_var('TOIL_WES_BROKER_URL', "a URL to a RabbitMQ broker for Celery")(test_item)
     return test_item
 
@@ -686,7 +661,7 @@ def needs_wes_server(test_item: MT) -> MT:
     Use as a decorator before test classes or methods to run only if a WES
     server is available to run against.
     """
-    test_item = _mark_test('wes_server', test_item)
+    test_item = _mark_test('wes_server', needs_online(test_item))
 
     wes_url = os.environ.get('TOIL_WES_ENDPOINT')
     if not wes_url:
@@ -744,7 +719,7 @@ def needs_fetchable_appliance(test_item: MT) -> MT:
     the Toil appliance Docker image is able to be downloaded from the Internet.
     """
 
-    test_item = _mark_test('fetchable_appliance', test_item)
+    test_item = _mark_test('fetchable_appliance', needs_online(test_item))
     if os.getenv('TOIL_SKIP_DOCKER', '').lower() == 'true':
         return unittest.skip('Skipping docker test.')(test_item)
     try:
@@ -765,9 +740,7 @@ def integrative(test_item: MT) -> MT:
     Use this to decorate integration tests so as to skip them during regular builds.
 
     We define integration tests as A) involving other, non-Toil software components
-    that we develop and/or B) having a higher cost (time or money). Note that brittleness
-    does not qualify a test for being integrative. Neither does involvement of external
-    services such as AWS, since that would cover most of Toil's test.
+    that we develop and/or B) having a higher cost (time or money).
     """
     test_item = _mark_test('integrative', test_item)
     if os.getenv('TOIL_TEST_INTEGRATIVE', '').lower() == 'true':
@@ -797,11 +770,13 @@ methodNamePartRegex = re.compile('^[a-zA-Z_0-9]+$')
 @contextmanager
 def timeLimit(seconds: int) -> Generator[None, None, None]:
     """
-    http://stackoverflow.com/a/601168
-    Use to limit the execution time of a function. Raises an exception if the execution of the
-    function takes more than the specified amount of time.
+    Use to limit the execution time of a function.
+
+    Raises an exception if the execution of the function takes more than the
+    specified amount of time. See <http://stackoverflow.com/a/601168>.
 
     :param seconds: maximum allowable time, in seconds
+    
     >>> import time
     >>> with timeLimit(2):
     ...    time.sleep(1)