toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/job.py

@@ -51,11 +51,11 @@ from typing import (
 from urllib.error import HTTPError
 from urllib.parse import urlsplit, unquote, urljoin
 
-from toil import memoize
-
 import dill
 from configargparse import ArgParser
 
+from toil.lib.memoize import memoize
+from toil.lib.misc import StrPath
 from toil.lib.io import is_remote_url
 
 if sys.version_info < (3, 11):
@@ -140,8 +140,8 @@ class FilesDownloadedStoppingPointReached(DebugStoppingPointReached):
     """
 
     def __init__(
-        self, message, host_and_job_paths: Optional[list[tuple[str, str]]] = None
-    ):
+        self, message: str, host_and_job_paths: Optional[list[tuple[str, str]]] = None
+    ) -> None:
         super().__init__(message)
 
         # Save the host and user-code-visible paths of files, in case we're
@@ -307,7 +307,7 @@ def parse_accelerator(
                 parsed["model"] = possible_description
     elif isinstance(spec, dict):
         # It's a dict, so merge with the defaults.
-        parsed.update(spec)
+        parsed.update(cast(AcceleratorRequirement, spec))
         # TODO: make sure they didn't misspell keys or something
     else:
         raise TypeError(
@@ -816,7 +816,7 @@ class JobDescription(Requirer):
 
     def __init__(
         self,
-        requirements: Mapping[str, Union[int, str, bool]],
+        requirements: Mapping[str, Union[int, str, float, bool, list]],
         jobName: str,
         unitName: Optional[str] = "",
         displayName: Optional[str] = "",
@@ -1767,7 +1767,7 @@ class Job:
         # Holds flags set by set_debug_flag()
         self._debug_flags: set[str] = set()
 
-    def __str__(self):
+    def __str__(self) -> str:
         """
         Produce a useful logging string to identify this Job and distinguish it
         from its JobDescription.
@@ -1812,16 +1812,16 @@ class Job:
         return self.description.disk
 
     @disk.setter
-    def disk(self, val):
+    def disk(self, val: int) -> None:
         self.description.disk = val
 
     @property
-    def memory(self):
+    def memory(self) -> int:
         """The maximum number of bytes of memory the job will require to run."""
         return self.description.memory
 
     @memory.setter
-    def memory(self, val):
+    def memory(self, val: int) -> None:
         self.description.memory = val
 
     @property
@@ -1830,7 +1830,7 @@ class Job:
         return self.description.cores
 
     @cores.setter
-    def cores(self, val):
+    def cores(self, val: int) -> None:
         self.description.cores = val
 
     @property
@@ -1848,11 +1848,11 @@ class Job:
         return self.description.preemptible
 
     @deprecated(new_function_name="preemptible")
-    def preemptable(self):
+    def preemptable(self) -> bool:
         return self.description.preemptible
 
     @preemptible.setter
-    def preemptible(self, val):
+    def preemptible(self, val: bool) -> None:
         self.description.preemptible = val
 
     @property
@@ -1865,13 +1865,13 @@ class Job:
         return self.description.files_to_use
 
     @files_to_use.setter
-    def files_to_use(self, val: set[FileID]):
+    def files_to_use(self, val: set[FileID]) -> None:
         self.description.files_to_use = val
 
-    def add_to_files_to_use(self, val: FileID):
+    def add_to_files_to_use(self, val: FileID) -> None:
         self.description.files_to_use.add(val)
 
-    def remove_from_files_to_use(self, val: FileID):
+    def remove_from_files_to_use(self, val: FileID) -> None:
         self.description.files_to_use.remove(val)
 
     def assignConfig(self, config: Config) -> None:
@@ -2296,7 +2296,7 @@ class Job:
 
         return {self._registry[jid] for jid in roots}
 
-    def checkJobGraphConnected(self):
+    def checkJobGraphConnected(self) -> None:
         """
         :raises toil.job.JobGraphDeadlockException: if :func:`toil.job.Job.getRootJobs` does \
         not contain exactly one root job.
@@ -2312,7 +2312,7 @@ class Job:
                 "Graph does not contain exactly one" " root job: %s" % rootJobs
             )
 
-    def checkJobGraphAcylic(self):
+    def checkJobGraphAcylic(self) -> None:
         """
         :raises toil.job.JobGraphDeadlockException: if the connected component \
         of jobs containing this job contains any cycles of child/followOn dependencies \
@@ -2486,10 +2486,13 @@ class Job:
         """Used to setup and run Toil workflow."""
 
         @staticmethod
-        def getDefaultArgumentParser(jobstore_as_flag: bool = False) -> ArgumentParser:
+        def getDefaultArgumentParser(jobstore_as_flag: bool = False) -> ArgParser:
             """
             Get argument parser with added toil workflow options.
 
+            This is the Right Way to get an argument parser in a Toil Python
+            workflow.
+
             :param jobstore_as_flag: make the job store option a --jobStore flag instead of a required jobStore positional argument.
             :returns: The argument parser used by a toil workflow with added Toil options.
             """
@@ -2499,7 +2502,7 @@ class Job:
 
         @staticmethod
         def getDefaultOptions(
-            jobStore: Optional[str] = None, jobstore_as_flag: bool = False
+            jobStore: Optional[StrPath] = None, jobstore_as_flag: bool = False
         ) -> Namespace:
             """
             Get default options for a toil workflow.
@@ -2520,9 +2523,9 @@ class Job:
             )
             arguments = []
             if jobstore_as_flag and jobStore is not None:
-                arguments = ["--jobstore", jobStore]
+                arguments = ["--jobstore", str(jobStore)]
             if not jobstore_as_flag and jobStore is not None:
-                arguments = [jobStore]
+                arguments = [str(jobStore)]
             return parser.parse_args(args=arguments)
 
         @staticmethod
@@ -2534,6 +2537,13 @@ class Job:
             Adds the default toil options to an :mod:`optparse` or :mod:`argparse`
             parser object.
 
+            Consider using :meth:`getDefaultArgumentParser` instead, which will
+            produce a parser of the correct class to use Toil's config file and
+            environment variables. If ther parser passed here is just an
+            :class:`argparse.ArgumentParser` and not a
+            :class:`configargparse.ArgParser`, the Toil config file and
+            environment variables will not be respected.
+
             :param parser: Options object to add toil options to.
             :param jobstore_as_flag: make the job store option a --jobStore flag instead of a required jobStore positional argument.
             """
@@ -2571,13 +2581,13 @@ class Job:
 
         def __init__(
             self,
-            memory=None,
-            cores=None,
-            disk=None,
-            accelerators=None,
-            preemptible=None,
-            unitName=None,
-        ):
+            memory: Optional[ParseableIndivisibleResource] = None,
+            cores: Optional[ParseableDivisibleResource] = None,
+            disk: Optional[ParseableIndivisibleResource] = None,
+            accelerators: Optional[ParseableAcceleratorRequirement] = None,
+            preemptible: Optional[ParseableFlag] = None,
+            unitName: Optional[str] = "",
+        ) -> None:
             """
             Memory, core and disk requirements are specified identically to as in \
             :func:`toil.job.Job.__init__`.
@@ -2603,7 +2613,7 @@ class Job:
             self.hostID = None
 
         @abstractmethod
-        def start(self, job: "Job") -> Any:
+        def start(self, job: "ServiceHostJob") -> Any:
             """
             Start the service.
 
@@ -2616,7 +2626,7 @@ class Job:
             """
 
         @abstractmethod
-        def stop(self, job: "Job") -> None:
+        def stop(self, job: "ServiceHostJob") -> None:
             """
             Stops the service. Function can block until complete.
 
@@ -3141,45 +3151,54 @@ class Job:
             startClock = ResourceMonitor.get_total_cpu_time()
         baseDir = os.getcwd()
 
-        yield
-
-        if "download_only" in self._debug_flags:
-            # We should stop right away
-            logger.debug("Job did not stop itself after downloading files; stopping.")
-            raise DebugStoppingPointReached()
-
-        # If the job is not a checkpoint job, add the promise files to delete
-        # to the list of jobStoreFileIDs to delete
-        # TODO: why is Promise holding a global list here???
-        if not self.checkpoint:
-            for jobStoreFileID in Promise.filesToDelete:
-                # Make sure to wrap the job store ID in a FileID object so the file store will accept it
-                # TODO: talk directly to the job store here instead.
-                fileStore.deleteGlobalFile(FileID(jobStoreFileID, 0))
-        else:
-            # Else copy them to the job description to delete later
-            self.description.checkpointFilesToDelete = list(Promise.filesToDelete)
-        Promise.filesToDelete.clear()
-        # Now indicate the asynchronous update of the job can happen
-        fileStore.startCommit(jobState=True)
-        # Change dir back to cwd dir, if changed by job (this is a safety issue)
-        if os.getcwd() != baseDir:
-            os.chdir(baseDir)
-        # Finish up the stats
-        if stats is not None:
-            totalCpuTime, totalMemoryUsage = (
-                ResourceMonitor.get_total_cpu_time_and_memory_usage()
-            )
-            stats.jobs.append(
-                Expando(
-                    time=str(time.time() - startTime),
-                    clock=str(totalCpuTime - startClock),
-                    class_name=self._jobName(),
-                    memory=str(totalMemoryUsage),
-                    requested_cores=str(self.cores),
-                    disk=str(fileStore.get_disk_usage()),
+        succeeded = False
+        try:
+            yield
+
+            if "download_only" in self._debug_flags:
+                # We should stop right away
+                logger.debug("Job did not stop itself after downloading files; stopping.")
+                raise DebugStoppingPointReached()
+
+            # If the job is not a checkpoint job, add the promise files to delete
+            # to the list of jobStoreFileIDs to delete
+            # TODO: why is Promise holding a global list here???
+            if not self.checkpoint:
+                for jobStoreFileID in Promise.filesToDelete:
+                    # Make sure to wrap the job store ID in a FileID object so the file store will accept it
+                    # TODO: talk directly to the job store here instead.
+                    fileStore.deleteGlobalFile(FileID(jobStoreFileID, 0))
+            else:
+                # Else copy them to the job description to delete later
+                self.description.checkpointFilesToDelete = list(Promise.filesToDelete)
+            Promise.filesToDelete.clear()
+            # Now indicate the asynchronous update of the job can happen
+            fileStore.startCommit(jobState=True)
+
+            succeeded = True
+        finally:
+            # Change dir back to cwd dir, if changed by job (this is a safety issue)
+            if os.getcwd() != baseDir:
+                os.chdir(baseDir)
+            # Finish up the stats
+            if stats is not None:
+                totalCpuTime, total_memory_kib = (
+                    ResourceMonitor.get_total_cpu_time_and_memory_usage()
+                )
+                stats.jobs.append(
+                    # TODO: We represent everything as strings in the stats
+                    # even though the JSON transport can take bools and floats.
+                    Expando(
+                        start=str(startTime),
+                        time=str(time.time() - startTime),
+                        clock=str(totalCpuTime - startClock),
+                        class_name=self._jobName(),
+                        memory=str(total_memory_kib),
+                        requested_cores=str(self.cores), # TODO: Isn't this really consumed cores?
+                        disk=str(fileStore.get_disk_usage()),
+                        succeeded=str(succeeded),
+                    )
                 )
-            )
 
     def _runner(
         self,
@@ -3294,7 +3313,9 @@ class FunctionWrappingJob(Job):
     Job used to wrap a function. In its `run` method the wrapped function is called.
     """
 
-    def __init__(self, userFunction, *args, **kwargs):
+    def __init__(
+        self, userFunction: Callable[[...], Any], *args: Any, **kwargs: Any
+    ) -> None:
         """
         :param callable userFunction: The function to wrap. It will be called with ``*args`` and
                ``**kwargs`` as arguments.
@@ -3317,7 +3338,9 @@ class FunctionWrappingJob(Job):
                 list(zip(argSpec.args[-len(argSpec.defaults) :], argSpec.defaults))
             )
 
-        def resolve(key, default=None, dehumanize=False):
+        def resolve(
+            key, default: Optional[Any] = None, dehumanize: bool = False
+        ) -> Any:
             try:
                 # First, try constructor arguments, ...
                 value = kwargs.pop(key)
@@ -3351,7 +3374,7 @@ class FunctionWrappingJob(Job):
         self._args = args
         self._kwargs = kwargs
 
-    def _getUserFunction(self):
+    def _getUserFunction(self) -> Callable[..., Any]:
         logger.debug(
             "Loading user function %s from module %s.",
             self.userFunctionName,
@@ -3360,14 +3383,14 @@ class FunctionWrappingJob(Job):
         userFunctionModule = self._loadUserModule(self.userFunctionModule)
         return getattr(userFunctionModule, self.userFunctionName)
 
-    def run(self, fileStore):
+    def run(self, fileStore: "AbstractFileStore") -> Any:
         userFunction = self._getUserFunction()
         return userFunction(*self._args, **self._kwargs)
 
-    def getUserScript(self):
+    def getUserScript(self) -> str:
         return self.userFunctionModule
 
-    def _jobName(self):
+    def _jobName(self) -> str:
         return ".".join(
             (
                 self.__class__.__name__,
@@ -3405,10 +3428,10 @@ class JobFunctionWrappingJob(FunctionWrappingJob):
     """
 
     @property
-    def fileStore(self):
+    def fileStore(self) -> "AbstractFileStore":
         return self._fileStore
 
-    def run(self, fileStore):
+    def run(self, fileStore: "AbstractFileStore") -> Any:
         userFunction = self._getUserFunction()
         rValue = userFunction(*((self,) + tuple(self._args)), **self._kwargs)
         return rValue
@@ -3504,7 +3527,7 @@ class EncapsulatedJob(Job):
     the same value after A or A.encapsulate() has been run.
     """
 
-    def __init__(self, job, unitName=None):
+    def __init__(self, job: Optional[Job], unitName: Optional[str] = None) -> None:
         """
         :param toil.job.Job job: the job to encapsulate.
         :param str unitName: human-readable name to identify this job instance.
@@ -3538,7 +3561,7 @@ class EncapsulatedJob(Job):
             self.encapsulatedJob = None
             self.encapsulatedFollowOn = None
 
-    def addChild(self, childJob):
+    def addChild(self, childJob: Job) -> Job:
         if self.encapsulatedFollowOn is None:
             raise RuntimeError(
                 "Children cannot be added to EncapsulatedJob while it is running"
@@ -3554,7 +3577,7 @@ class EncapsulatedJob(Job):
             self.encapsulatedFollowOn, service, parentService=parentService
         )
 
-    def addFollowOn(self, followOnJob):
+    def addFollowOn(self, followOnJob: Job) -> Job:
         if self.encapsulatedFollowOn is None:
             raise RuntimeError(
                 "Follow-ons cannot be added to EncapsulatedJob while it is running"
@@ -3977,7 +4000,7 @@ class CombineImportsJob(Job):
         self._d = d
         super().__init__(**kwargs)
 
-    def run(self, file_store: AbstractFileStore) -> Promised[Dict[str, FileID]]:
+    def run(self, file_store: "AbstractFileStore") -> Promised[Dict[str, FileID]]:
         """
         Merge the dicts
         """
@@ -4032,7 +4055,7 @@ class WorkerImportJob(Job):
                 path_to_fileid[file] = imported
         return path_to_fileid
 
-    def run(self, file_store: AbstractFileStore) -> Promised[Dict[str, FileID]]:
+    def run(self, file_store: "AbstractFileStore") -> Promised[Dict[str, FileID]]:
         """
         Import the workflow inputs and then create and run the workflow.
         :return: Promise of workflow outputs
@@ -4068,7 +4091,7 @@ class ImportsJob(Job):
         self._import_worker_disk = import_worker_disk
 
     def run(
-        self, file_store: AbstractFileStore
+        self, file_store: "AbstractFileStore"
     ) -> Tuple[Promised[Dict[str, FileID]], Dict[str, FileMetadata]]:
         """
         Import the workflow inputs and then create and run the workflow.
@@ -4098,7 +4121,8 @@ class ImportsJob(Job):
                     # schedule the individual file
                     per_batch_files.append(filename)
                 file_batches.append(per_batch_files)
-                # reset batching calculation
+                # reset batch to empty
+                per_batch_files = []
                 per_batch_size = 0
             else:
                 per_batch_size += filesize
@@ -4265,7 +4289,7 @@ class PromisedRequirement:
     C = B.addChildFn(h, cores=PromisedRequirement(lambda x: 2*x, B.rv()))
     """
 
-    def __init__(self, valueOrCallable, *args):
+    def __init__(self, valueOrCallable: Any, *args: Any) -> None:
         """
         Initialize this Promised Requirement.
 
@@ -4289,7 +4313,7 @@ class PromisedRequirement:
         self._func = dill.dumps(func)
         self._args = list(args)
 
-    def getValue(self):
+    def getValue(self) -> Any:
         """Return PromisedRequirement value."""
         func = dill.loads(self._func)
         return func(*self._args)

toil/jobStores/abstractJobStore.py

@@ -1686,17 +1686,39 @@ class AbstractJobStore(ABC):
         sharedFileName: str,
         encoding: Optional[str] = None,
         errors: Optional[str] = None,
-    ) -> ContextManager[IO[bytes]]:
+    ) -> Union[ContextManager[IO[str]], ContextManager[IO[bytes]]]:
         return self.read_shared_file_stream(sharedFileName, encoding, errors)
 
+    @overload
     @abstractmethod
     @contextmanager
     def read_shared_file_stream(
         self,
         shared_file_name: str,
-        encoding: Optional[str] = None,
+        encoding: str,
+        errors: Optional[str] = None,
+    ) -> Iterator[IO[str]]:
+        """If encoding is specified, then a text file handle is provided."""
+
+    @overload
+    @abstractmethod
+    @contextmanager
+    def read_shared_file_stream(
+        self,
+        shared_file_name: str,
+        encoding: Literal[None] = None,
         errors: Optional[str] = None,
     ) -> Iterator[IO[bytes]]:
+        """If no encoding is provided, then a bytest file handle is provided."""
+
+    @abstractmethod
+    @contextmanager
+    def read_shared_file_stream(
+        self,
+        shared_file_name: str,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+    ) -> Union[Iterator[IO[str]], Iterator[IO[bytes]]]:
         """
         Returns a context manager yielding a readable file handle to the global file referenced
         by the given name.
@@ -1711,7 +1733,6 @@ class AbstractJobStore(ABC):
                 are the same as for open(). Defaults to 'strict' when an encoding is specified.
 
         :return: a context manager yielding a readable file handle
-        :rtype: Iterator[IO[bytes]]
         """
         raise NotImplementedError()
 

toil/jobStores/aws/jobStore.py

@@ -637,30 +637,61 @@ class AWSJobStore(AbstractJobStore):
         else:
             super()._default_export_file(otherCls, file_id, uri)
 
+    ###
+    # URL access implementation
+    ###
+
+    # URL access methods aren't used by the rest of the job store methods.
+
     @classmethod
     def _url_exists(cls, url: ParseResult) -> bool:
         try:
-            get_object_for_url(url, existing=True)
+            try:
+                get_object_for_url(url, existing=True, anonymous=True)
+            except PermissionError:
+                # If we can't look anonymously, log in
+                get_object_for_url(url, existing=True)
             return True
         except FileNotFoundError:
             # Not a file
-            # Might be a directory.
+            # Might be a directory. Or we might not have access to know.
+            # See if it's a directory.
             return cls._get_is_directory(url)
 
     @classmethod
     def _get_size(cls, url: ParseResult) -> int:
-        return get_object_for_url(url, existing=True).content_length
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+        except PermissionError:
+            src_obj = get_object_for_url(url, existing=True)
+        return src_obj.content_length
 
     @classmethod
     def _read_from_url(cls, url: ParseResult, writable):
-        srcObj = get_object_for_url(url, existing=True)
-        srcObj.download_fileobj(writable)
-        return (srcObj.content_length, False)  # executable bit is always False
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            src_obj.download_fileobj(writable)
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                src_obj.download_fileobj(writable)
+            else:
+                raise
+        return (src_obj.content_length, False)  # executable bit is always False
 
     @classmethod
     def _open_url(cls, url: ParseResult) -> IO[bytes]:
-        src_obj = get_object_for_url(url, existing=True)
-        response = src_obj.get()
+        try:
+            src_obj = get_object_for_url(url, existing=True, anonymous=True)
+            response = src_obj.get()
+        except Exception as e:
+            if isinstance(e, PermissionError) or (isinstance(e, ClientError) and get_error_status(e) == 403):
+                # The object setup or the download does not have permission. Try again with a login.
+                src_obj = get_object_for_url(url, existing=True)
+                response = src_obj.get()
+            else:
+                raise
         # We should get back a response with a stream in 'Body'
         if "Body" not in response:
             raise RuntimeError(f"Could not fetch body stream for {url}")
@@ -670,6 +701,7 @@ class AWSJobStore(AbstractJobStore):
     def _write_to_url(
         cls, readable, url: ParseResult, executable: bool = False
     ) -> None:
+        # Don't try to do anonympus writes.
         dstObj = get_object_for_url(url)
 
         logger.debug("Uploading %s", dstObj.key)
@@ -684,13 +716,17 @@ class AWSJobStore(AbstractJobStore):
 
     @classmethod
     def _list_url(cls, url: ParseResult) -> list[str]:
-        return list_objects_for_url(url)
+        try:
+            return list_objects_for_url(url, anonymous=True)
+        except PermissionError:
+            return list_objects_for_url(url)
+        
 
     @classmethod
     def _get_is_directory(cls, url: ParseResult) -> bool:
         # We consider it a directory if anything is in it.
         # TODO: Can we just get the first item and not the whole list?
-        return len(list_objects_for_url(url)) > 0
+        return len(cls._list_url(url)) > 0
 
     @classmethod
     def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:

toil/jobStores/fileJobStore.py

@@ -395,6 +395,7 @@ class FileJobStore(AbstractJobStore):
 
     @classmethod
     def _url_exists(cls, url: ParseResult) -> bool:
+        # Note that broken symlinks will not be shown to exist.
         return os.path.exists(cls._extract_path_from_url(url))
 
     @classmethod
@@ -771,8 +772,31 @@ class FileJobStore(AbstractJobStore):
             ) as f:
                 yield f
 
+    @overload
     @contextmanager
-    def read_shared_file_stream(self, shared_file_name, encoding=None, errors=None):
+    def read_shared_file_stream(
+        self,
+        shared_file_name: str,
+        encoding: str,
+        errors: Optional[str] = None,
+    ) -> Iterator[IO[str]]: ...
+
+    @overload
+    @contextmanager
+    def read_shared_file_stream(
+        self,
+        shared_file_name: str,
+        encoding: Literal[None] = None,
+        errors: Optional[str] = None,
+    ) -> Iterator[IO[bytes]]: ...
+
+    @contextmanager
+    def read_shared_file_stream(
+        self,
+        shared_file_name: str,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+    ) -> Union[Iterator[IO[bytes]], Iterator[IO[str]]]:
         self._requireValidSharedFileName(shared_file_name)
         try:
             with open(