toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/cwl/cwltoil.py

@@ -17,43 +17,43 @@
 
 # For an overview of how this all works, see discussion in
 # docs/architecture.rst
-import argparse
 import base64
 import copy
 import datetime
 import errno
 import functools
+import glob
+import io
 import json
 import logging
 import os
+import pprint
 import shutil
 import socket
 import stat
 import sys
-import tempfile
 import textwrap
-import urllib
 import uuid
+from tempfile import NamedTemporaryFile, TemporaryFile, gettempdir
 from threading import Thread
-from typing import (
-    IO,
-    Any,
-    Callable,
-    Dict,
-    Iterator,
-    List,
-    Mapping,
-    MutableMapping,
-    MutableSequence,
-    Optional,
-    TextIO,
-    Tuple,
-    Type,
-    TypeVar,
-    Union,
-    cast,
-)
-from urllib.parse import ParseResult, quote, unquote, urlparse, urlsplit
+from typing import (IO,
+                    Any,
+                    Callable,
+                    Dict,
+                    Iterator,
+                    List,
+                    Mapping,
+                    MutableMapping,
+                    MutableSequence,
+                    Optional,
+                    Sequence,
+                    TextIO,
+                    Tuple,
+                    Type,
+                    TypeVar,
+                    Union,
+                    cast)
+from urllib.parse import quote, unquote, urlparse, urlsplit
 
 import cwl_utils.errors
 import cwl_utils.expression
@@ -66,35 +66,30 @@ import cwltool.load_tool
 import cwltool.main
 import cwltool.resolver
 import schema_salad.ref_resolver
+from configargparse import SUPPRESS, ArgParser, Namespace
 from cwltool.loghandler import _logger as cwllogger
 from cwltool.loghandler import defaultStreamHandler
 from cwltool.mpi import MpiConfig
 from cwltool.mutation import MutationManager
 from cwltool.pathmapper import MapperEnt, PathMapper
-from cwltool.process import (
-    Process,
-    add_sizes,
-    compute_checksums,
-    fill_in_defaults,
-    shortname,
-)
+from cwltool.process import (Process,
+                             add_sizes,
+                             compute_checksums,
+                             fill_in_defaults,
+                             shortname)
 from cwltool.secrets import SecretStore
-from cwltool.software_requirements import (
-    DependenciesConfiguration,
-    get_container_from_software_requirements,
-)
+from cwltool.software_requirements import (DependenciesConfiguration,
+                                           get_container_from_software_requirements)
 from cwltool.stdfsaccess import StdFsAccess, abspath
-from cwltool.utils import (
-    CWLObjectType,
-    CWLOutputType,
-    DirectoryType,
-    adjustDirObjs,
-    aslist,
-    downloadHttpFile,
-    get_listing,
-    normalizeFilesDirs,
-    visit_class,
-)
+from cwltool.utils import (CWLObjectType,
+                           CWLOutputType,
+                           DirectoryType,
+                           adjustDirObjs,
+                           aslist,
+                           downloadHttpFile,
+                           get_listing,
+                           normalizeFilesDirs,
+                           visit_class)
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Names
 from schema_salad.exceptions import ValidationException
@@ -103,28 +98,31 @@ from schema_salad.sourceline import SourceLine
 from typing_extensions import Literal
 
 from toil.batchSystems.registry import DEFAULT_BATCH_SYSTEM
-from toil.common import Config, Toil, addOptions
-from toil.cwl.utils import (
-    CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION,
-    CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE,
-    download_structure,
-    visit_cwl_class_and_reduce,
-)
+from toil.common import Toil, addOptions
+from toil.cwl import check_cwltool_version
+
+check_cwltool_version()
+from toil.cwl.utils import (CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION,
+                            CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE,
+                            download_structure,
+                            get_from_structure,
+                            visit_cwl_class_and_reduce)
 from toil.exceptions import FailedJobsException
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import AcceleratorRequirement, Job, Promise, Promised, unwrap
-from toil.jobStores.abstractJobStore import AbstractJobStore, NoSuchFileException
+from toil.jobStores.abstractJobStore import (AbstractJobStore,
+                                             NoSuchFileException)
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.jobStores.utils import JobStoreUnavailableException, generate_locator
+from toil.lib.io import mkdtemp
 from toil.lib.threading import ExceptionalThread
 from toil.statsAndLogging import DEFAULT_LOGLEVEL
-from toil.version import baseVersion
 
 logger = logging.getLogger(__name__)
 
 # Find the default temporary directory
-DEFAULT_TMPDIR = tempfile.gettempdir()
+DEFAULT_TMPDIR = gettempdir()
 # And compose a CWL-style default prefix inside it.
 # We used to not put this inside anything and we would drop loads of temp
 # directories in the current directory and leave them there.
@@ -349,16 +347,24 @@ class ResolveSource:
 
     def __repr__(self) -> str:
         """Allow for debug printing."""
-        try:
-            return "ResolveSource(" + repr(self.resolve()) + ")"
-        except Exception:
-            return (
-                f"ResolveSource({self.name}, {self.input}, {self.source_key}, "
-                f"{self.promise_tuples})"
-            )
+
+        parts = [f"source key {self.source_key}"]
+
+        if "pickValue" in self.input:
+            parts.append(f"pick value {self.input['pickValue']} from")
+
+        if isinstance(self.promise_tuples, list):
+            names = [n for n, _ in self.promise_tuples]
+            parts.append(f"names {names} in promises")
+        else:
+            name, _ = self.promise_tuples
+            parts.append(f"name {name} in promise")
+
+        return f"ResolveSource({', '.join(parts)})"
 
     def resolve(self) -> Any:
         """First apply linkMerge then pickValue if either present."""
+
         result: Optional[Any] = None
         if isinstance(self.promise_tuples, list):
             result = self.link_merge(
@@ -382,6 +388,7 @@ class ResolveSource:
 
         :param values: result of step
         """
+
         link_merge_type = self.input.get("linkMerge", "merge_nested")
 
         if link_merge_type == "merge_nested":
@@ -409,6 +416,7 @@ class ResolveSource:
                        without modification.
         :return:
         """
+
         pick_value_type = cast(str, self.input.get("pickValue"))
 
         if pick_value_type is None:
@@ -425,6 +433,11 @@ class ResolveSource:
 
         if pick_value_type == "first_non_null":
             if len(result) < 1:
+                logger.error(
+                    "Could not find non-null entry for %s:\n%s",
+                    self.name,
+                    pprint.pformat(self.promise_tuples),
+                )
                 raise cwl_utils.errors.WorkflowException(
                     "%s: first_non_null operator found no non-null values" % self.name
                 )
@@ -479,6 +492,11 @@ class StepValueFrom:
         self.req = req
         self.container_engine = container_engine
 
+    def __repr__(self) -> str:
+        """Allow for debug printing."""
+
+        return f"StepValueFrom({self.expr}, {self.source}, {self.req}, {self.container_engine})"
+
     def eval_prep(
         self, step_inputs: CWLObjectType, file_store: AbstractFileStore
     ) -> None:
@@ -551,6 +569,11 @@ class DefaultWithSource:
         self.default = default
         self.source = source
 
+    def __repr__(self) -> str:
+        """Allow for debug printing."""
+
+        return f"DefaultWithSource({self.default}, {self.source})"
+
     def resolve(self) -> Any:
         """
         Determine the final input value when the time is right.
@@ -573,6 +596,11 @@ class JustAValue:
         """Store the value."""
         self.val = val
 
+    def __repr__(self) -> str:
+        """Allow for debug printing."""
+
+        return f"JustAValue({self.val})"
+
     def resolve(self) -> Any:
         """Return the value."""
         return self.val
@@ -654,6 +682,8 @@ class ToilPathMapper(PathMapper):
                streaming on, and returns a file: URI to where the file or
                directory has been downloaded to. Meant to be a partially-bound
                version of toil_get_file().
+        :param referenced_files: List of CWL File and Directory objects, which can have their locations set as both
+               virtualized and absolute local paths
         """
         self.get_file = get_file
         self.stage_listing = stage_listing
@@ -675,28 +705,29 @@ class ToilPathMapper(PathMapper):
         This is called on each File or Directory CWL object. The Files and
         Directories all have "location" fields. For the Files, these are from
         upload_file(), and for the Directories, these are from
-        upload_directory(), with their children being assigned
-        locations based on listing the Directories using ToilFsAccess.
+        upload_directory() or cwltool internally. With upload_directory(), they and their children will be assigned
+        locations based on listing the Directories using ToilFsAccess. With cwltool, locations will be set as absolute
+        paths.
 
         :param obj: The CWL File or Directory to process
 
         :param stagedir: The base path for target paths to be generated under,
-        except when a File or Directory has an overriding parent directory in
-        dirname
+            except when a File or Directory has an overriding parent directory in
+            dirname
 
         :param basedir: The directory from which relative paths should be
-        resolved; used as the base directory for the StdFsAccess that generated
-        the listing being processed.
+            resolved; used as the base directory for the StdFsAccess that generated
+            the listing being processed.
 
         :param copy: If set, use writable types for Files and Directories.
 
         :param staged: Starts as True at the top of the recursion. Set to False
-        when entering a directory that we can actually download, so we don't
-        stage files and subdirectories separately from the directory as a
-        whole. Controls the staged flag on generated mappings, and therefore
-        whether files and directories are actually placed at their mapped-to
-        target locations. If stage_listing is True, we will leave this True
-        throughout and stage everything.
+            when entering a directory that we can actually download, so we don't
+            stage files and subdirectories separately from the directory as a
+            whole. Controls the staged flag on generated mappings, and therefore
+            whether files and directories are actually placed at their mapped-to
+            target locations. If stage_listing is True, we will leave this True
+            throughout and stage everything.
 
         Produces one MapperEnt for every unique location for a File or
         Directory. These MapperEnt objects are instructions to cwltool's
@@ -807,6 +838,14 @@ class ToilPathMapper(PathMapper):
                     # We can't really make the directory. Maybe we are
                     # exporting from the leader and it doesn't matter.
                     resolved = location
+            elif location.startswith("/"):
+                # Test if path is an absolute local path
+                # Does not check if the path is relative
+                # While Toil encodes paths into a URL with ToilPathMapper,
+                # something called internally in cwltool may return an absolute path
+                # ex: if cwltool calls itself internally in command_line_tool.py,
+                # it collects outputs with collect_output, and revmap_file will use its own internal pathmapper
+                resolved = location
             else:
                 raise RuntimeError("Unsupported location: " + location)
 
@@ -883,7 +922,6 @@ class ToilPathMapper(PathMapper):
                         )
                     else:
                         deref = ab
-
                     if deref.startswith("file:"):
                         deref = schema_salad.ref_resolver.uri_file_path(deref)
                     if urlsplit(deref).scheme in ["http", "https"]:
@@ -1027,8 +1065,6 @@ class ToilCommandLineTool(ToilTool, cwltool.command_line_tool.CommandLineTool):
 class ToilExpressionTool(ToilTool, cwltool.command_line_tool.ExpressionTool):
     """Subclass the cwltool expression tool to provide the custom ToilPathMapper."""
 
-    pass
-
 
 def toil_make_tool(
     toolpath_object: CommentedMap,
@@ -1047,10 +1083,7 @@ def toil_make_tool(
     return cwltool.workflow.default_make_tool(toolpath_object, loadingContext)
 
 
-# This should really be Dict[str, Union[str, "DirectoryContents"]], but we
-# can't say that until https://github.com/python/mypy/issues/731 is fixed
-# because it's recursive.
-DirectoryContents = Dict[str, Union[str, Dict[str, Any]]]
+DirectoryContents = Dict[str, Union[str, "DirectoryContents"]]
 
 
 def check_directory_dict_invariants(contents: DirectoryContents) -> None:
@@ -1080,9 +1113,8 @@ def decode_directory(
     None), and the deduplication key string that uniquely identifies the
     directory.
     """
-    assert dir_path.startswith(
-        "toildir:"
-    ), f"Cannot decode non-directory path: {dir_path}"
+    if not dir_path.startswith("toildir:"):
+        raise RuntimeError(f"Cannot decode non-directory path: {dir_path}")
 
     # We will decode the directory and then look inside it
 
@@ -1203,7 +1235,8 @@ class ToilFsAccess(StdFsAccess):
 
                 logger.debug("ToilFsAccess downloading %s to %s", cache_key, temp_dir)
 
-                # Save it all into this new temp directory
+                # Save it all into this new temp directory.
+                # Guaranteed to fill it with real files and not symlinks.
                 download_structure(self.file_store, {}, {}, contents, temp_dir)
 
                 # Make sure we use the same temp directory if we go traversing
@@ -1233,7 +1266,7 @@ class ToilFsAccess(StdFsAccess):
                     logger.debug(
                         "ToilFsAccess fetching directory %s from a JobStore", path
                     )
-                    dest_dir = tempfile.mkdtemp()
+                    dest_dir = mkdtemp()
 
                     # Recursively fetch all the files in the directory.
                     def download_to(url: str, dest: str) -> None:
@@ -1256,7 +1289,7 @@ class ToilFsAccess(StdFsAccess):
                     logger.debug("ToilFsAccess fetching file %s from a JobStore", path)
                     # Try to grab it with a jobstore implementation, and save it
                     # somewhere arbitrary.
-                    dest_file = tempfile.NamedTemporaryFile(delete=False)
+                    dest_file = NamedTemporaryFile(delete=False)
                     AbstractJobStore.read_from_url(path, dest_file)
                     dest_file.close()
                     self.dir_to_download[path] = dest_file.name
@@ -1271,72 +1304,160 @@ class ToilFsAccess(StdFsAccess):
         return destination
 
     def glob(self, pattern: str) -> List[str]:
-        # We know this falls back on _abs
-        return super().glob(pattern)
+        parse = urlparse(pattern)
+        if parse.scheme == "file":
+            pattern = os.path.abspath(unquote(parse.path))
+        elif parse.scheme == "":
+            pattern = os.path.abspath(pattern)
+        else:
+            raise RuntimeError(f"Cannot efficiently support globbing on {parse.scheme} URIs")
+
+        # Actually do the glob
+        return [schema_salad.ref_resolver.file_uri(f) for f in glob.glob(pattern)]
 
     def open(self, fn: str, mode: str) -> IO[Any]:
-        # TODO: Also implement JobStore-supported URLs through JobStore methods.
-        # We know this falls back on _abs
-        return super().open(fn, mode)
+        if "w" in mode or "x" in mode or "+" in mode or "a" in mode:
+            raise RuntimeError(f"Mode {mode} for opening {fn} involves writing")
+
+        parse = urlparse(fn)
+        if parse.scheme in ["", "file"]:
+            # Handle local files
+            return open(self._abs(fn), mode)
+        elif parse.scheme == "toildir":
+            contents, subpath, cache_key = decode_directory(fn)
+            if cache_key in self.dir_to_download:
+                # This is already available locally, so fall back on the local copy
+                return open(self._abs(fn), mode)
+            else:
+                # We need to get the URI out of the virtual directory
+                if subpath is None:
+                    raise RuntimeError(f"{fn} is a toildir directory")
+                uri = get_from_structure(contents, subpath)
+                if not isinstance(uri, str):
+                    raise RuntimeError(f"{fn} does not point to a file")
+                # Recurse on that URI
+                return self.open(uri, mode)
+        elif parse.scheme == "toilfile":
+            if self.file_store is None:
+                raise RuntimeError("URL requires a file store: " + fn)
+            # Streaming access to Toil file store files requires being inside a
+            # context manager, which we can't require. So we need to download
+            # the file.
+            return open(self._abs(fn), mode)
+        else:
+            # This should be supported by a job store.
+            byte_stream = AbstractJobStore.open_url(fn)
+            if 'b' in mode:
+                # Pass stream along in binary
+                return byte_stream
+            else:
+                # Wrap it in a text decoder
+                return io.TextIOWrapper(byte_stream, encoding='utf-8')
 
     def exists(self, path: str) -> bool:
         """Test for file existence."""
-        # toil's _abs() throws errors when files are not found and cwltool's _abs() does not
-        try:
-            # TODO: Also implement JobStore-supported URLs through JobStore methods.
-            return os.path.exists(self._abs(path))
-        except NoSuchFileException:
-            return False
+        parse = urlparse(path)
+        if parse.scheme in ["", "file"]:
+            # Handle local files
+            # toil's _abs() throws errors when files are not found and cwltool's _abs() does not
+            try:
+                return os.path.exists(self._abs(path))
+            except NoSuchFileException:
+                return False
+        elif parse.scheme == "toildir":
+            contents, subpath, cache_key = decode_directory(path)
+            if subpath is None:
+                # The toildir directory itself exists
+                return True
+            uri = get_from_structure(contents, subpath)
+            if uri is None:
+                # It's not in the virtual directory, so it doesn't exist
+                return False
+            if isinstance(uri, dict):
+                # Actually it's a subdirectory, so it exists.
+                return True
+            # We recurse and poll the URI directly to make sure it really exists
+            return self.exists(uri)
+        elif parse.scheme == "toilfile":
+            # TODO: we assume CWL can't call deleteGlobalFile and so the file always exists
+            return True
+        else:
+            # This should be supported by a job store.
+            return AbstractJobStore.url_exists(path)
 
     def size(self, path: str) -> int:
-        # This should avoid _abs for things actually in the file store, to
-        # prevent multiple downloads as in
-        # https://github.com/DataBiosphere/toil/issues/3665
-        if path.startswith("toilfile:"):
-            if self.file_store is None:
-                raise RuntimeError("URL requires a file store: " + path)
-            return self.file_store.getGlobalFileSize(
-                FileID.unpack(path[len("toilfile:") :])
-            )
-        elif path.startswith("toildir:"):
+        parse = urlparse(path)
+        if parse.scheme in ["", "file"]:
+            return os.stat(self._abs(path)).st_size
+        elif parse.scheme == "toildir":
             # Decode its contents, the path inside it to the file (if any), and
             # the key to use for caching the directory.
-            here, subpath, cache_key = decode_directory(path)
+            contents, subpath, cache_key = decode_directory(path)
 
             # We can't get the size of just a directory.
-            assert subpath is not None, f"Attempted to check size of directory {path}"
-
-            for part in subpath.split("/"):
-                # Follow the path inside the directory contents.
-                here = cast(DirectoryContents, here[part])
+            if subpath is None:
+                raise RuntimeError(f"Attempted to check size of directory {path}")
 
-            # We ought to end up with a toilfile: URI.
-            assert isinstance(here, str), f"Did not find a file at {path}"
-            assert here.startswith(
-                "toilfile:"
-            ), f"Did not find a filestore file at {path}"
+            uri = get_from_structure(contents, subpath)
 
-            return self.size(here)
+            # We ought to end up with a URI.
+            if not isinstance(uri, str):
+                raise RuntimeError(f"Did not find a file at {path}")
+            return self.size(uri)
+        elif parse.scheme == "toilfile":
+            if self.file_store is None:
+                raise RuntimeError("URL requires a file store: " + path)
+            return self.file_store.getGlobalFileSize(
+                FileID.unpack(path[len("toilfile:") :])
+            )
         else:
-            # TODO: Also implement JobStore-supported URLs through JobStore methods.
-            # We know this falls back on _abs
-            return super().size(path)
+            # This should be supported by a job store.
+            size = AbstractJobStore.get_size(path)
+            if size is None:
+                # get_size can be unimplemented or unavailable
+                raise RuntimeError(f"Could not get size of {path}")
+            return size
 
     def isfile(self, fn: str) -> bool:
         parse = urlparse(fn)
-        if parse.scheme in ["toilfile", "toildir", "file", ""]:
-            # We know this falls back on _abs
-            return super().isfile(fn)
+        if parse.scheme in ["file", ""]:
+            return os.path.isfile(self._abs(fn))
+        elif parse.scheme == "toilfile":
+            # TODO: we assume CWL can't call deleteGlobalFile and so the file always exists
+            return True
+        elif parse.scheme == "toildir":
+            contents, subpath, cache_key = decode_directory(fn)
+            if subpath is None:
+                # This is the toildir directory itself
+                return False
+            found = get_from_structure(contents, subpath)
+            # If we find a string, that's a file
+            # TODO: we assume CWL can't call deleteGlobalFile and so the file always exists
+            return isinstance(found, str)
         else:
-            return not AbstractJobStore.get_is_directory(fn)
+            return self.exists(fn) and not AbstractJobStore.get_is_directory(fn)
 
     def isdir(self, fn: str) -> bool:
+        logger.debug("ToilFsAccess checking type of %s", fn)
         parse = urlparse(fn)
-        if parse.scheme in ["toilfile", "toildir", "file", ""]:
-            # We know this falls back on _abs
-            return super().isdir(fn)
+        if parse.scheme in ["file", ""]:
+            return os.path.isdir(self._abs(fn))
+        elif parse.scheme == "toilfile":
+            return False
+        elif parse.scheme == "toildir":
+            contents, subpath, cache_key = decode_directory(fn)
+            if subpath is None:
+                # This is the toildir directory itself.
+                # TODO: We assume directories can't be deleted.
+                return True
+            found = get_from_structure(contents, subpath)
+            # If we find a dict, that's a directory.
+            # TODO: We assume directories can't be deleted.
+            return isinstance(found, dict)
         else:
-            return AbstractJobStore.get_is_directory(fn)
+            status = AbstractJobStore.get_is_directory(fn)
+            logger.debug("AbstractJobStore said: %s", status)
+            return status
 
     def listdir(self, fn: str) -> List[str]:
         # This needs to return full URLs for everything in the directory.
@@ -1344,12 +1465,25 @@ class ToilFsAccess(StdFsAccess):
         logger.debug("ToilFsAccess listing %s", fn)
 
         parse = urlparse(fn)
-        if parse.scheme in ["toilfile", "toildir", "file", ""]:
-            # Download the file or directory to a local path
+        if parse.scheme in ["file", ""]:
+            # Find the local path
             directory = self._abs(fn)
-
             # Now list it (it is probably a directory)
             return [abspath(quote(entry), fn) for entry in os.listdir(directory)]
+        elif parse.scheme == "toilfile":
+            raise RuntimeError(f"Cannot list a file: {fn}")
+        elif parse.scheme == "toildir":
+            contents, subpath, cache_key = decode_directory(fn)
+            here = contents
+            if subpath is not None:
+                got = get_from_structure(contents, subpath)
+                if got is None:
+                    raise RuntimeError(f"Cannot list nonexistent directory: {fn}")
+                if isinstance(got, str):
+                    raise RuntimeError(f"Cannot list file or dubdirectory of a file: {fn}")
+                here = got
+            # List all the things in here and make full URIs to them
+            return [os.path.join(fn, k) for k in here.keys()]
         else:
             return [
                 os.path.join(fn, entry.rstrip("/"))
@@ -1371,7 +1505,7 @@ def toil_get_file(
     file_store: AbstractFileStore,
     index: Dict[str, str],
     existing: Dict[str, str],
-    file_store_id: str,
+    uri: str,
     streamable: bool = False,
     streaming_allowed: bool = True,
     pipe_threads: Optional[List[Tuple[Thread, int]]] = None,
@@ -1388,28 +1522,28 @@ def toil_get_file(
 
     :param index: Maps from downloaded file path back to input Toil URI.
 
-    :param existing: Maps from file_store_id URI to downloaded file path.
+    :param existing: Maps from URI to downloaded file path.
 
-    :param file_store_id: The URI for the file to download.
+    :param uri: The URI for the file to download.
 
     :param streamable: If the file is has 'streamable' flag set
 
     :param streaming_allowed: If streaming is allowed
 
     :param pipe_threads: List of threads responsible for streaming the data
-    and open file descriptors corresponding to those files. Caller is responsible
-    to close the file descriptors (to break the pipes) and join the threads
+        and open file descriptors corresponding to those files. Caller is responsible
+        to close the file descriptors (to break the pipes) and join the threads
     """
     pipe_threads_real = pipe_threads or []
     # We can't use urlparse here because we need to handle the '_:' scheme and
     # urlparse sees that as a path and not a URI scheme.
-    if file_store_id.startswith("toildir:"):
+    if uri.startswith("toildir:"):
         # This is a file in a directory, or maybe a directory itself.
         # See ToilFsAccess and upload_directory.
         # We will go look for the actual file in the encoded directory
         # structure which will tell us where the toilfile: name for the file is.
 
-        parts = file_store_id[len("toildir:") :].split("/")
+        parts = uri[len("toildir:") :].split("/")
         contents = json.loads(
             base64.urlsafe_b64decode(parts[0].encode("utf-8")).decode("utf-8")
         )
@@ -1429,21 +1563,41 @@ def toil_get_file(
             download_structure(file_store, index, existing, contents, dest_path)
             # Return where we put it, but as a file:// URI
             return schema_salad.ref_resolver.file_uri(dest_path)
-    elif file_store_id.startswith("toilfile:"):
-        # This is a plain file with no context.
+    elif uri.startswith("_:"):
+        # Someone is asking us for an empty temp directory.
+        # We need to check this before the file path case because urlsplit()
+        # will call this a path with no scheme.
+        dest_path = file_store.getLocalTempDir()
+        return schema_salad.ref_resolver.file_uri(dest_path)
+    elif uri.startswith("file:") or urlsplit(uri).scheme == "":
+        # There's a file: scheme or no scheme, and we know this isn't a _: URL.
+
+        # We need to support file: URIs and local paths, because we might be
+        # involved in moving files around on the local disk when uploading
+        # things after a job. We might want to catch cases where a leader
+        # filesystem file URI leaks in here, but we can't, so we just rely on
+        # the rest of the code to be correct.
+        return uri
+    else:
+        # This is a toilfile: uri or other remote URI
         def write_to_pipe(
-            file_store: AbstractFileStore, pipe_name: str, file_store_id: FileID
+            file_store: AbstractFileStore, pipe_name: str, uri: str
         ) -> None:
             try:
                 with open(pipe_name, "wb") as pipe:
-                    with file_store.jobStore.read_file_stream(file_store_id) as fi:
-                        file_store.logAccess(file_store_id)
-                        chunk_sz = 1024
-                        while True:
-                            data = fi.read(chunk_sz)
-                            if not data:
-                                break
-                            pipe.write(data)
+                    if uri.startswith("toilfile:"):
+                        # Stream from the file store
+                        file_store_id = FileID.unpack(uri[len("toilfile:") :])
+                        with file_store.readGlobalFileStream(file_store_id) as fi:
+                            chunk_sz = 1024
+                            while True:
+                                data = fi.read(chunk_sz)
+                                if not data:
+                                    break
+                                pipe.write(data)
+                    else:
+                        # Stream from some other URI
+                        AbstractJobStore.read_from_url(uri, pipe)
             except OSError as e:
                 # The other side of the pipe may have been closed by the
                 # reading thread, which is OK.
@@ -1456,7 +1610,7 @@ def toil_get_file(
             and not isinstance(file_store.jobStore, FileJobStore)
         ):
             logger.debug(
-                "Streaming file %s", FileID.unpack(file_store_id[len("toilfile:") :])
+                "Streaming file %s", uri
             )
             src_path = file_store.getLocalTempFileName()
             os.mkfifo(src_path)
@@ -1465,42 +1619,39 @@ def toil_get_file(
                 args=(
                     file_store,
                     src_path,
-                    FileID.unpack(file_store_id[len("toilfile:") :]),
+                    uri,
                 ),
             )
             th.start()
             pipe_threads_real.append((th, os.open(src_path, os.O_RDONLY)))
         else:
-            src_path = file_store.readGlobalFile(
-                FileID.unpack(file_store_id[len("toilfile:") :]), symlink=True
-            )
-
-        # TODO: shouldn't we be using these as a cache?
-        index[src_path] = file_store_id
-        existing[file_store_id] = src_path
+            # We need to do a real file
+            if uri in existing:
+                # Already did it
+                src_path = existing[uri]
+            else:
+                if uri.startswith("toilfile:"):
+                    # Download from the file store
+                    file_store_id = FileID.unpack(uri[len("toilfile:") :])
+                    src_path = file_store.readGlobalFile(
+                        file_store_id, symlink=True
+                    )
+                else:
+                    # Download from the URI via the job store.
+
+                    # Figure out where it goes.
+                    src_path = file_store.getLocalTempFileName()
+                    # Open that path exclusively to make sure we created it
+                    with open(src_path, 'xb') as fh:
+                        # Download into the file
+                       size, executable = AbstractJobStore.read_from_url(uri, fh)
+                       if executable:
+                           # Set the execute bit in the file's permissions
+                           os.chmod(src_path, os.stat(src_path).st_mode | stat.S_IXUSR)
+
+        index[src_path] = uri
+        existing[uri] = src_path
         return schema_salad.ref_resolver.file_uri(src_path)
-    elif file_store_id.startswith("_:"):
-        # Someone is asking us for an empty temp directory.
-        # We need to check this before the file path case because urlsplit()
-        # will call this a path with no scheme.
-        dest_path = file_store.getLocalTempDir()
-        return schema_salad.ref_resolver.file_uri(dest_path)
-    elif file_store_id.startswith("file:") or urlsplit(file_store_id).scheme == "":
-        # There's a file: scheme or no scheme, and we know this isn't a _: URL.
-
-        # We need to support file: URIs and local paths, because we might be
-        # involved in moving files around on the local disk when uploading
-        # things after a job. We might want to catch cases where a leader
-        # filesystem file URI leaks in here, but we can't, so we just rely on
-        # the rest of the code to be correct.
-        return file_store_id
-    else:
-        raise RuntimeError(
-            f"Cannot obtain file {file_store_id} while on host "
-            f"{socket.gethostname()}; all imports must happen on the "
-            f"leader!"
-        )
-
 
 def write_file(
     writeFunc: Callable[[str], FileID],
@@ -1557,7 +1708,9 @@ def import_files(
     existing: Dict[str, str],
     cwl_object: Optional[CWLObjectType],
     skip_broken: bool = False,
+    skip_remote: bool = False,
     bypass_file_store: bool = False,
+    log_level: int = logging.DEBUG
 ) -> None:
     """
     Prepare all files and directories.
@@ -1579,28 +1732,41 @@ def import_files(
     Also does some miscelaneous normalization.
 
     :param import_function: The function used to upload a URI and get a
-    Toil FileID for it.
+        Toil FileID for it.
 
     :param fs_access: the CWL FS access object we use to access the filesystem
-    to find files to import. Needs to support the URI schemes used.
+        to find files to import. Needs to support the URI schemes used.
 
     :param fileindex: Forward map to fill in from file URI to Toil storage
-    location, used by write_file to deduplicate writes.
+        location, used by write_file to deduplicate writes.
 
     :param existing: Reverse map to fill in from Toil storage location to file
-    URI. Not read from.
+        URI. Not read from.
 
     :param cwl_object: CWL tool (or workflow order) we are importing files for
 
     :param skip_broken: If True, when files can't be imported because they e.g.
-    don't exist, leave their locations alone rather than failing with an error.
+        don't exist, leave their locations alone rather than failing with an error.
+
+    :param skp_remote: If True, leave remote URIs in place instead of importing
+        files.
 
     :param bypass_file_store: If True, leave file:// URIs in place instead of
-    importing files and directories.
+        importing files and directories.
+
+    :param log_level: Log imported files at the given level.
     """
     tool_id = cwl_object.get("id", str(cwl_object)) if cwl_object else ""
 
     logger.debug("Importing files for %s", tool_id)
+    logger.debug("Importing files in %s", cwl_object)
+
+    def import_and_log(url: str) -> FileID:
+        """
+        Upload a file and log that we are doing so.
+        """
+        logger.log(log_level, "Loading %s...", url)
+        return import_function(url)
 
     # We need to upload all files to the Toil filestore, and encode structure
     # recursively into all Directories' locations. But we cannot safely alter
@@ -1700,7 +1866,7 @@ def import_files(
 
             # Upload the file itself, which will adjust its location.
             upload_file(
-                import_function, fileindex, existing, rec, skip_broken=skip_broken
+                import_and_log, fileindex, existing, rec, skip_broken=skip_broken, skip_remote=skip_remote
             )
 
             # Make a record for this file under its name
@@ -1805,11 +1971,16 @@ def upload_file(
     existing: Dict[str, str],
     file_metadata: CWLObjectType,
     skip_broken: bool = False,
+    skip_remote: bool = False
 ) -> None:
     """
-    Update a file object so that the location is a reference to the toil file store.
+    Update a file object so that the file will be accessible from another machine.
 
-    Write the file object to the file store if necessary.
+    Uploads local files to the Toil file store, and sets their location to a
+    reference to the toil file store.
+
+    Unless skip_remote is set, downloads remote files into the file store and
+    sets their locations to references into the file store as well.
     """
     location = cast(str, file_metadata["location"])
     if (
@@ -1832,7 +2003,10 @@ def upload_file(
             return
         else:
             raise cwl_utils.errors.WorkflowException("File is missing: %s" % location)
-    file_metadata["location"] = write_file(uploadfunc, fileindex, existing, location)
+
+    if location.startswith("file://") or not skip_remote:
+        # This is a local file, or we also need to download and re-upload remote files
+        file_metadata["location"] = write_file(uploadfunc, fileindex, existing, location)
 
     logger.debug("Sending file at: %s", file_metadata["location"])
 
@@ -1866,6 +2040,7 @@ class CWLNamedJob(Job):
         memory: Union[int, str, None] = "1GiB",
         disk: Union[int, str, None] = "1MiB",
         accelerators: Optional[List[AcceleratorRequirement]] = None,
+        preemptible: Optional[bool] = None,
         tool_id: Optional[str] = None,
         parent_name: Optional[str] = None,
         subjob_name: Optional[str] = None,
@@ -1910,6 +2085,7 @@ class CWLNamedJob(Job):
             memory=memory,
             disk=disk,
             accelerators=accelerators,
+            preemptible=preemptible,
             unitName=unit_name,
             displayName=display_name,
             local=local,
@@ -1941,12 +2117,15 @@ def toilStageFiles(
     cwljob: Union[CWLObjectType, List[CWLObjectType]],
     outdir: str,
     destBucket: Union[str, None] = None,
+    log_level: int = logging.DEBUG
 ) -> None:
     """
     Copy input files out of the global file store and update location and path.
 
     :param destBucket: If set, export to this base URL instead of to the local
            filesystem.
+
+    :param log_level: Log each file transfered at the given level.
     """
 
     def _collectDirEntries(
@@ -1986,7 +2165,6 @@ def toilStageFiles(
         stage_listing=True,
     )
     for _, p in pm.items():
-        logger.debug("Staging output: %s", p)
         if p.staged:
             # We're supposed to copy/expose something.
             # Note that we have to handle writable versions of everything
@@ -2008,7 +2186,7 @@ def toilStageFiles(
                         "CreateFile",
                         "CreateWritableFile",
                     ]:  # TODO: CreateFile for buckets is not under testing
-                        with tempfile.NamedTemporaryFile() as f:
+                        with NamedTemporaryFile() as f:
                             # Make a file with the right contents
                             f.write(file_id_or_contents.encode("utf-8"))
                             f.close()
@@ -2027,39 +2205,63 @@ def toilStageFiles(
                         # At the end we should get a direct toilfile: URI
                         file_id_or_contents = cast(str, here)
 
+                    # This might be an e.g. S3 URI now
+                    if not file_id_or_contents.startswith("toilfile:"):
+                        # We need to import it so we can export it.
+                        # TODO: Use direct S3 to S3 copy on exports as well
+                        file_id_or_contents = (
+                            "toilfile:"
+                            + toil.import_file(file_id_or_contents, symlink=False).pack()
+                        )
+
                     if file_id_or_contents.startswith("toilfile:"):
                         # This is something we can export
-                        destUrl = "/".join(s.strip("/") for s in [destBucket, baseName])
-                        toil.exportFile(
+                        # TODO: Do we need to urlencode the parts before sending them to S3?
+                        dest_url = "/".join(s.strip("/") for s in [destBucket, baseName])
+                        logger.log(log_level, "Saving %s...", dest_url)
+                        toil.export_file(
                             FileID.unpack(file_id_or_contents[len("toilfile:") :]),
-                            destUrl,
+                            dest_url,
                         )
                     # TODO: can a toildir: "file" get here?
             else:
-                # We are saving to the filesystem so we only really need exportFile for actual files.
+                # We are saving to the filesystem.
+                dest_url = "file://" + quote(p.target)
+
+                # We only really need export_file for actual files.
                 if not os.path.exists(p.target) and p.type in [
                     "Directory",
                     "WritableDirectory",
                 ]:
                     os.makedirs(p.target)
-                if not os.path.exists(p.target) and p.type in ["File", "WritableFile"]:
-                    if p.resolved.startswith("toilfile:"):
-                        # We can actually export this
-                        os.makedirs(os.path.dirname(p.target), exist_ok=True)
-                        toil.exportFile(
-                            FileID.unpack(p.resolved[len("toilfile:") :]),
-                            "file://" + p.target,
-                        )
-                    elif p.resolved.startswith("/"):
+                if p.type in ["File", "WritableFile"]:
+                    if p.resolved.startswith("/"):
                         # Probably staging and bypassing file store. Just copy.
+                        logger.log(log_level, "Saving %s...", dest_url)
                         os.makedirs(os.path.dirname(p.target), exist_ok=True)
                         shutil.copyfile(p.resolved, p.target)
-                    # TODO: can a toildir: "file" get here?
-                if not os.path.exists(p.target) and p.type in [
+                    else:
+                        uri = p.resolved
+                        if not uri.startswith("toilfile:"):
+                            # We need to import so we can export
+                            uri = (
+                                "toilfile:"
+                                + toil.import_file(uri, symlink=False).pack()
+                            )
+
+                        # Actually export from the file store
+                        logger.log(log_level, "Saving %s...", dest_url)
+                        os.makedirs(os.path.dirname(p.target), exist_ok=True)
+                        toil.export_file(
+                            FileID.unpack(uri[len("toilfile:") :]),
+                            dest_url,
+                        )
+                if p.type in [
                     "CreateFile",
                     "CreateWritableFile",
                 ]:
                     # We just need to make a file with particular contents
+                    logger.log(log_level, "Saving %s...", dest_url)
                     os.makedirs(os.path.dirname(p.target), exist_ok=True)
                     with open(p.target, "wb") as n:
                         n.write(p.resolved.encode("utf-8"))
@@ -2078,6 +2280,7 @@ def toilStageFiles(
             # Make the location point to the place we put this thing on the
             # local filesystem.
             f["location"] = schema_salad.ref_resolver.file_uri(mapped_location.target)
+            f["path"] = mapped_location.target
 
         if "contents" in f:
             del f["contents"]
@@ -2182,7 +2385,7 @@ class CWLJob(CWLNamedJob):
 
         accelerators: Optional[List[AcceleratorRequirement]] = None
         if req.get("cudaDeviceCount", 0) > 0:
-            # There's a CUDARequirement
+            # There's a CUDARequirement, which cwltool processed for us
             # TODO: How is cwltool deciding what value to use between min and max?
             accelerators = [
                 {
@@ -2192,14 +2395,62 @@ class CWLJob(CWLNamedJob):
                 }
             ]
 
+        # cwltool doesn't handle http://arvados.org/cwl#UsePreemptible as part
+        # of its resource logic so we have to do it manually.
+        #
+        # Note that according to
+        # https://github.com/arvados/arvados/blob/48a0d575e6de34bcda91c489e4aa98df291a8cca/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml#L345
+        # this can only be a literal boolean! cwltool doesn't want to evaluate
+        # expressions in the value for us like it does for CUDARequirement
+        # which has a schema which allows for CWL expressions:
+        # https://github.com/common-workflow-language/cwltool/blob/1573509eea2faa3cd1dc959224e52ff1d796d3eb/cwltool/extensions.yml#L221
+        #
+        # By default we have default preemptibility.
+        preemptible: Optional[bool] = None
+        preemptible_req, _ = tool.get_requirement(
+            "http://arvados.org/cwl#UsePreemptible"
+        )
+        if preemptible_req:
+            if "usePreemptible" not in preemptible_req:
+                # If we have a requirement it has to have the value
+                raise ValidationException(
+                    f"Unacceptable syntax for http://arvados.org/cwl#UsePreemptible: "
+                    f"expected key usePreemptible but got: {preemptible_req}"
+                )
+            parsed_value = preemptible_req["usePreemptible"]
+            if isinstance(parsed_value, str) and (
+                "$(" in parsed_value or "${" in parsed_value
+            ):
+                # Looks like they tried to use an expression
+                raise ValidationException(
+                    f"Unacceptable value for usePreemptible in http://arvados.org/cwl#UsePreemptible: "
+                    f"expected true or false but got what appears to be an expression: {repr(parsed_value)}. "
+                    f"Note that expressions are not allowed here by Arvados's schema."
+                )
+            if not isinstance(parsed_value, bool):
+                # If we have a value it has to be a bool flag
+                raise ValidationException(
+                    f"Unacceptable value for usePreemptible in http://arvados.org/cwl#UsePreemptible: "
+                    f"expected true or false but got: {repr(parsed_value)}"
+                )
+            preemptible = parsed_value
+
+        # We always need space for the temporary files for the job
+        total_disk = cast(int, req["tmpdirSize"]) * (2**20)
+        if not getattr(runtime_context, "bypass_file_store", False):
+            # If using the Toil file store, we also need space for the output
+            # files, which may need to be stored locally and copied off the
+            # node.
+            total_disk += cast(int, req["outdirSize"]) * (2**20)
+        # If not using the Toil file store, output files just go directly to
+        # their final homes their space doesn't need to be accounted per-job.
+
         super().__init__(
             cores=req["cores"],
             memory=int(req["ram"] * (2**20)),
-            disk=int(
-                (cast(int, req["tmpdirSize"]) * (2**20))
-                + (cast(int, req["outdirSize"]) * (2**20))
-            ),
+            disk=int(total_disk),
             accelerators=accelerators,
+            preemptible=preemptible,
             tool_id=self.cwltool.tool["id"],
             parent_name=parent_name,
             local=isinstance(tool, cwltool.command_line_tool.ExpressionTool),
@@ -2265,7 +2516,7 @@ class CWLJob(CWLNamedJob):
         cwllogger.removeHandler(defaultStreamHandler)
         cwllogger.setLevel(logger.getEffectiveLevel())
 
-        logger.debug("Loaded order: %s", self.cwljob)
+        logger.debug("Loaded order:\n%s", self.cwljob)
 
         cwljob = resolve_dict_w_promises(self.cwljob, file_store)
 
@@ -2354,6 +2605,13 @@ class CWLJob(CWLNamedJob):
                 streaming_allowed=runtime_context.streaming_allowed,
             )
 
+        # Collect standard output and standard error somewhere if they don't go to files.
+        # We need to keep two FDs to these because cwltool will close what we give it.
+        default_stdout = TemporaryFile()
+        runtime_context.default_stdout = os.fdopen(os.dup(default_stdout.fileno()), 'wb')
+        default_stderr = TemporaryFile()
+        runtime_context.default_stderr = os.fdopen(os.dup(default_stderr.fileno()), 'wb')
+
         process_uuid = uuid.uuid4()  # noqa F841
         started_at = datetime.datetime.now()  # noqa F841
 
@@ -2362,13 +2620,34 @@ class CWLJob(CWLNamedJob):
         logger.debug("Running tool %s with order: %s", self.cwltool, self.cwljob)
 
         runtime_context.name = self.description.unitName
-        output, status = ToilSingleJobExecutor().execute(
-            process=self.cwltool,
-            job_order_object=cwljob,
-            runtime_context=runtime_context,
-            logger=cwllogger,
-        )
-        ended_at = datetime.datetime.now()  # noqa F841
+
+        status = "did_not_run"
+        try:
+            output, status = ToilSingleJobExecutor().execute(
+                process=self.cwltool,
+                job_order_object=cwljob,
+                runtime_context=runtime_context,
+                logger=cwllogger,
+            )
+        finally:
+            ended_at = datetime.datetime.now()  # noqa F841
+
+            # Log any output/error data
+            default_stdout.seek(0, os.SEEK_END)
+            if default_stdout.tell() > 0:
+                default_stdout.seek(0)
+                file_store.log_user_stream(self.description.unitName + '.stdout', default_stdout)
+                if status != "success":
+                    default_stdout.seek(0)
+                    logger.error("Failed command standard output:\n%s", default_stdout.read().decode("utf-8", errors="replace"))
+            default_stderr.seek(0, os.SEEK_END)
+            if default_stderr.tell():
+                default_stderr.seek(0)
+                file_store.log_user_stream(self.description.unitName + '.stderr', default_stderr)
+                if status != "success":
+                    default_stderr.seek(0)
+                    logger.error("Failed command standard error:\n%s", default_stderr.read().decode("utf-8", errors="replace"))
+
         if status != "success":
             raise cwl_utils.errors.WorkflowException(status)
 
@@ -2395,6 +2674,8 @@ class CWLJob(CWLNamedJob):
 
         logger.debug("Emitting output: %s", output)
 
+        file_store.log_to_leader(f"CWL step complete: {runtime_context.name}")
+
         # metadata[process_uuid] = {
         #     'started_at': started_at,
         #     'ended_at': ended_at,
@@ -2782,6 +3063,10 @@ class CWLWorkflow(CWLNamedJob):
         if self.conditional.is_false(cwljob):
             return self.conditional.skipped_outputs()
 
+        # Apply default values set in the workflow
+        fs_access = ToilFsAccess(self.runtime_context.basedir, file_store=file_store)
+        fill_in_defaults(self.cwlwf.tool["inputs"], cwljob, fs_access)
+
         # `promises` dict
         # from: each parameter (workflow input or step output)
         #   that may be used as a "source" for a step input workflow output
@@ -2844,6 +3129,10 @@ class CWLWorkflow(CWLNamedJob):
                                     get_container_engine(self.runtime_context),
                                 )
 
+                            logger.debug(
+                                "Value will come from %s", jobobj.get(key, None)
+                            )
+
                         conditional = Conditional(
                             expression=step.tool.get("when"),
                             outputs=step.tool["out"],
@@ -3042,8 +3331,8 @@ def scan_for_unsupported_requirements(
     :param tool: The CWL tool to check for unsupported requirements.
 
     :param bypass_file_store: True if the Toil file store is not being used to
-    transport files between nodes, and raw origin node file:// URIs are exposed
-    to tools instead.
+        transport files between nodes, and raw origin node file:// URIs are exposed
+        to tools instead.
 
     """
 
@@ -3080,24 +3369,31 @@ def determine_load_listing(
     DIRECTORY_NAME is any variable name) set to one of the following three
     options:
 
-        no_listing: DIRECTORY_NAME.listing will be undefined.
-            e.g. inputs.DIRECTORY_NAME.listing == unspecified
+    1. no_listing: DIRECTORY_NAME.listing will be undefined.
+        e.g.
+
+            inputs.DIRECTORY_NAME.listing == unspecified
+
+    2. shallow_listing: DIRECTORY_NAME.listing will return a list one level
+        deep of DIRECTORY_NAME's contents.
+        e.g.
+
+            inputs.DIRECTORY_NAME.listing == [items in directory]
+             inputs.DIRECTORY_NAME.listing[0].listing == undefined
+             inputs.DIRECTORY_NAME.listing.length == # of items in directory
 
-        shallow_listing: DIRECTORY_NAME.listing will return a list one level
-                         deep of DIRECTORY_NAME's contents.
-            e.g. inputs.DIRECTORY_NAME.listing == [items in directory]
-                 inputs.DIRECTORY_NAME.listing[0].listing == undefined
-                 inputs.DIRECTORY_NAME.listing.length == # of items in directory
+    3. deep_listing: DIRECTORY_NAME.listing will return a list of the entire
+        contents of DIRECTORY_NAME.
+        e.g.
 
-        deep_listing: DIRECTORY_NAME.listing will return a list of the entire
-                      contents of DIRECTORY_NAME.
-            e.g. inputs.DIRECTORY_NAME.listing == [items in directory]
-                 inputs.DIRECTORY_NAME.listing[0].listing == [items
-                      in subdirectory if it exists and is the first item listed]
-                 inputs.DIRECTORY_NAME.listing.length == # of items in directory
+            inputs.DIRECTORY_NAME.listing == [items in directory]
+            inputs.DIRECTORY_NAME.listing[0].listing == [items in subdirectory
+            if it exists and is the first item listed]
+            inputs.DIRECTORY_NAME.listing.length == # of items in directory
 
-    See: https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingRequirement
-         https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingEnum
+    See
+    https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingRequirement
+    and https://www.commonwl.org/v1.1/CommandLineTool.html#LoadListingEnum
 
     DIRECTORY_NAME.listing should be determined first from loadListing.
     If that's not specified, from LoadListingRequirement.
@@ -3209,6 +3505,20 @@ usage_message = "\n\n" + textwrap.dedent(
     ]
 )
 
+def get_options(args: List[str]) -> Namespace:
+    """
+    Parse given args and properly add non-Toil arguments into the cwljob of the Namespace.
+    :param args: List of args from command line
+    :return: options namespace
+    """
+    parser = ArgParser()
+    addOptions(parser, jobstore_as_flag=True, cwl=True)
+    options: Namespace
+    options, cwl_options = parser.parse_known_args(args)
+    options.cwljob.extend(cwl_options)
+
+    return options
+
 
 def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
     """Run the main loop for toil-cwl-runner."""
@@ -3218,334 +3528,20 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
     if args is None:
         args = sys.argv[1:]
 
-    config = Config()
-    config.disableChaining = True
-    config.cwl = True
-    parser = argparse.ArgumentParser()
-    addOptions(parser, config, jobstore_as_flag=True)
-    parser.add_argument("cwltool", type=str)
-    parser.add_argument("cwljob", nargs=argparse.REMAINDER)
-
-    parser.add_argument("--not-strict", action="store_true")
-    parser.add_argument(
-        "--enable-dev",
-        action="store_true",
-        help="Enable loading and running development versions of CWL",
-    )
-    parser.add_argument(
-        "--enable-ext",
-        action="store_true",
-        help="Enable loading and running 'cwltool:' extensions to the CWL standards.",
-        default=False,
-    )
-    parser.add_argument("--quiet", dest="quiet", action="store_true", default=False)
-    parser.add_argument("--basedir", type=str)  # TODO: Might be hard-coded?
-    parser.add_argument("--outdir", type=str, default=os.getcwd())
-    parser.add_argument("--version", action="version", version=baseVersion)
-    parser.add_argument(
-        "--log-dir",
-        type=str,
-        default="",
-        help="Log your tools stdout/stderr to this location outside of container",
-    )
-    dockergroup = parser.add_mutually_exclusive_group()
-    dockergroup.add_argument(
-        "--user-space-docker-cmd",
-        help="(Linux/OS X only) Specify a user space docker command (like "
-        "udocker or dx-docker) that will be used to call 'pull' and 'run'",
-    )
-    dockergroup.add_argument(
-        "--singularity",
-        action="store_true",
-        default=False,
-        help="Use Singularity runtime for running containers. "
-        "Requires Singularity v2.6.1+ and Linux with kernel version v3.18+ or "
-        "with overlayfs support backported.",
-    )
-    dockergroup.add_argument(
-        "--podman",
-        action="store_true",
-        default=False,
-        help="Use Podman runtime for running containers. ",
-    )
-    dockergroup.add_argument(
-        "--no-container",
-        action="store_true",
-        help="Do not execute jobs in a "
-        "Docker container, even when `DockerRequirement` "
-        "is specified under `hints`.",
-    )
-    dockergroup.add_argument(
-        "--leave-container",
-        action="store_false",
-        default=True,
-        help="Do not delete Docker container used by jobs after they exit",
-        dest="rm_container",
-    )
-    extra_dockergroup = parser.add_argument_group()
-    extra_dockergroup.add_argument(
-        "--custom-net",
-        help="Specify docker network name to pass to docker run command",
-    )
-    cidgroup = parser.add_argument_group(
-        "Options for recording the Docker container identifier into a file."
-    )
-    cidgroup.add_argument(
-        # Disabled as containerid is now saved by default
-        "--record-container-id",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-        dest="record_container_id",
-    )
-
-    cidgroup.add_argument(
-        "--cidfile-dir",
-        type=str,
-        help="Store the Docker container ID into a file in the specified directory.",
-        default=None,
-        dest="cidfile_dir",
-    )
-
-    cidgroup.add_argument(
-        "--cidfile-prefix",
-        type=str,
-        help="Specify a prefix to the container ID filename. "
-        "Final file name will be followed by a timestamp. "
-        "The default is no prefix.",
-        default=None,
-        dest="cidfile_prefix",
-    )
-
-    parser.add_argument(
-        "--preserve-environment",
-        type=str,
-        nargs="+",
-        help="Preserve specified environment variables when running"
-        " CommandLineTools",
-        metavar=("VAR1 VAR2"),
-        default=("PATH",),
-        dest="preserve_environment",
-    )
-    parser.add_argument(
-        "--preserve-entire-environment",
-        action="store_true",
-        help="Preserve all environment variable when running CommandLineTools.",
-        default=False,
-        dest="preserve_entire_environment",
-    )
-    parser.add_argument(
-        "--destBucket",
-        type=str,
-        help="Specify a cloud bucket endpoint for output files.",
-    )
-    parser.add_argument("--beta-dependency-resolvers-configuration", default=None)
-    parser.add_argument("--beta-dependencies-directory", default=None)
-    parser.add_argument("--beta-use-biocontainers", default=None, action="store_true")
-    parser.add_argument("--beta-conda-dependencies", default=None, action="store_true")
-    parser.add_argument(
-        "--tmpdir-prefix",
-        type=str,
-        help="Path prefix for temporary directories",
-        default=DEFAULT_TMPDIR_PREFIX,
-    )
-    parser.add_argument(
-        "--tmp-outdir-prefix",
-        type=str,
-        help="Path prefix for intermediate output directories",
-        default=DEFAULT_TMPDIR_PREFIX,
-    )
-    parser.add_argument(
-        "--force-docker-pull",
-        action="store_true",
-        default=False,
-        dest="force_docker_pull",
-        help="Pull latest docker image even if it is locally present",
-    )
-    parser.add_argument(
-        "--no-match-user",
-        action="store_true",
-        default=False,
-        help="Disable passing the current uid to `docker run --user`",
-    )
-    parser.add_argument(
-        "--no-read-only",
-        action="store_true",
-        default=False,
-        help="Do not set root directory in the container as read-only",
-    )
-    parser.add_argument(
-        "--strict-memory-limit",
-        action="store_true",
-        help="When running with "
-        "software containers and the Docker engine, pass either the "
-        "calculated memory allocation from ResourceRequirements or the "
-        "default of 1 gigabyte to Docker's --memory option.",
-    )
-    parser.add_argument(
-        "--strict-cpu-limit",
-        action="store_true",
-        help="When running with "
-        "software containers and the Docker engine, pass either the "
-        "calculated cpu allocation from ResourceRequirements or the "
-        "default of 1 core to Docker's --cpu option. "
-        "Requires docker version >= v1.13.",
-    )
-    parser.add_argument(
-        "--relax-path-checks",
-        action="store_true",
-        default=False,
-        help="Relax requirements on path names to permit "
-        "spaces and hash characters.",
-        dest="relax_path_checks",
-    )
-    parser.add_argument(
-        "--default-container",
-        help="Specify a default docker container that will be "
-        "used if the workflow fails to specify one.",
-    )
-    parser.add_argument(
-        "--disable-validate",
-        dest="do_validate",
-        action="store_false",
-        default=True,
-        help=argparse.SUPPRESS,
-    )
-    parser.add_argument(
-        "--fast-parser",
-        dest="fast_parser",
-        action="store_true",
-        default=False,
-        help=argparse.SUPPRESS,
-    )
-    checkgroup = parser.add_mutually_exclusive_group()
-    checkgroup.add_argument(
-        "--compute-checksum",
-        action="store_true",
-        default=True,
-        help="Compute checksum of contents while collecting outputs",
-        dest="compute_checksum",
-    )
-    checkgroup.add_argument(
-        "--no-compute-checksum",
-        action="store_false",
-        help="Do not compute checksum of contents while collecting outputs",
-        dest="compute_checksum",
-    )
-
-    parser.add_argument(
-        "--eval-timeout",
-        help="Time to wait for a Javascript expression to evaluate before giving "
-        "an error, default 20s.",
-        type=float,
-        default=20,
-    )
-    parser.add_argument(
-        "--overrides",
-        type=str,
-        default=None,
-        help="Read process requirement overrides from file.",
-    )
-
-    parser.add_argument(
-        "--mpi-config-file",
-        type=str,
-        default=None,
-        help="Platform specific configuration for MPI (parallel "
-        "launcher, its flag etc). See the cwltool README "
-        "section 'Running MPI-based tools' for details of the format: "
-        "https://github.com/common-workflow-language/cwltool#running-mpi-based-tools-that-need-to-be-launched",
-    )
-    parser.add_argument(
-        "--bypass-file-store",
-        action="store_true",
-        default=False,
-        help="Do not use Toil's file store and assume all "
-        "paths are accessible in place from all nodes.",
-        dest="bypass_file_store",
-    )
-    parser.add_argument(
-        "--disable-streaming",
-        action="store_true",
-        default=False,
-        help="Disable file streaming for files that have 'streamable' flag True",
-        dest="disable_streaming",
-    )
-
-    provgroup = parser.add_argument_group(
-        "Options for recording provenance information of the execution"
-    )
-    provgroup.add_argument(
-        "--provenance",
-        help="Save provenance to specified folder as a "
-        "Research Object that captures and aggregates "
-        "workflow execution and data products.",
-        type=str,
-    )
-
-    provgroup.add_argument(
-        "--enable-user-provenance",
-        default=False,
-        action="store_true",
-        help="Record user account info as part of provenance.",
-        dest="user_provenance",
-    )
-    provgroup.add_argument(
-        "--disable-user-provenance",
-        default=False,
-        action="store_false",
-        help="Do not record user account info in provenance.",
-        dest="user_provenance",
-    )
-    provgroup.add_argument(
-        "--enable-host-provenance",
-        default=False,
-        action="store_true",
-        help="Record host info as part of provenance.",
-        dest="host_provenance",
-    )
-    provgroup.add_argument(
-        "--disable-host-provenance",
-        default=False,
-        action="store_false",
-        help="Do not record host info in provenance.",
-        dest="host_provenance",
-    )
-    provgroup.add_argument(
-        "--orcid",
-        help="Record user ORCID identifier as part of "
-        "provenance, e.g. https://orcid.org/0000-0002-1825-0097 "
-        "or 0000-0002-1825-0097. Alternatively the environment variable "
-        "ORCID may be set.",
-        dest="orcid",
-        default=os.environ.get("ORCID", ""),
-        type=str,
-    )
-    provgroup.add_argument(
-        "--full-name",
-        help="Record full name of user as part of provenance, "
-        "e.g. Josiah Carberry. You may need to use shell quotes to preserve "
-        "spaces. Alternatively the environment variable CWL_FULL_NAME may "
-        "be set.",
-        dest="cwl_full_name",
-        default=os.environ.get("CWL_FULL_NAME", ""),
-        type=str,
-    )
-
-    # Parse all the options once.
-    options = parser.parse_args(args)
+    options = get_options(args)
 
     # Do cwltool setup
     cwltool.main.setup_schema(args=options, custom_schema_callback=None)
+    tmpdir_prefix = options.tmpdir_prefix = options.tmpdir_prefix or DEFAULT_TMPDIR_PREFIX
 
     # We need a workdir for the CWL runtime contexts.
-    if options.tmpdir_prefix != DEFAULT_TMPDIR_PREFIX:
+    if tmpdir_prefix != DEFAULT_TMPDIR_PREFIX:
         # if tmpdir_prefix is not the default value, move
         # workdir and the default job store under it
-        workdir = cwltool.utils.create_tmp_dir(options.tmpdir_prefix)
+        workdir = cwltool.utils.create_tmp_dir(tmpdir_prefix)
     else:
         # Use a directory in the default tmpdir
-        workdir = tempfile.mkdtemp()
+        workdir = mkdtemp()
     # Make sure workdir doesn't exist so it can be a job store
     os.rmdir(workdir)
 
@@ -3562,13 +3558,13 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
     options.do_validate = True
     options.pack = False
     options.print_subgraph = False
-    if options.tmpdir_prefix != DEFAULT_TMPDIR_PREFIX and options.workDir is None:
+    if tmpdir_prefix != DEFAULT_TMPDIR_PREFIX and options.workDir is None:
         # We need to override workDir because by default Toil will pick
         # somewhere under the system temp directory if unset, ignoring
         # --tmpdir-prefix.
         #
         # If set, workDir needs to exist, so we directly use the prefix
-        options.workDir = cwltool.utils.create_tmp_dir(options.tmpdir_prefix)
+        options.workDir = cwltool.utils.create_tmp_dir(tmpdir_prefix)
 
     if options.batchSystem == "kubernetes":
         # Containers under Kubernetes can only run in Singularity
@@ -3585,8 +3581,10 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
 
     logger.debug(f"Final job store {options.jobStore} and workDir {options.workDir}")
 
-    outdir = os.path.abspath(options.outdir)
-    tmp_outdir_prefix = os.path.abspath(options.tmp_outdir_prefix)
+    outdir = os.path.abspath(options.outdir or os.getcwd())
+    tmp_outdir_prefix = os.path.abspath(
+        options.tmp_outdir_prefix or DEFAULT_TMPDIR_PREFIX
+    )
 
     fileindex: Dict[str, str] = {}
     existing: Dict[str, str] = {}
@@ -3597,13 +3595,13 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
         dependencies_configuration = DependenciesConfiguration(options)
         job_script_provider = dependencies_configuration
 
-    options.default_container = None
     runtime_context = cwltool.context.RuntimeContext(vars(options))
     runtime_context.toplevel = True  # enable discovery of secondaryFiles
     runtime_context.find_default_container = functools.partial(
         find_default_container, options
     )
     runtime_context.workdir = workdir  # type: ignore[attr-defined]
+    runtime_context.outdir = outdir
     runtime_context.move_outputs = "leave"
     runtime_context.rm_tmpdir = False
     runtime_context.streaming_allowed = not options.disable_streaming
@@ -3621,12 +3619,16 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
         # Otherwise, if it takes a File with loadContents from a URL, we won't
         # be able to load the contents when we need to.
         runtime_context.make_fs_access = ToilFsAccess
+    if options.reference_inputs and options.bypass_file_store:
+        # We can't do both of these at the same time.
+        logger.error("Cannot reference inputs when bypassing the file store")
+        return 1
 
     loading_context = cwltool.main.setup_loadingContext(None, runtime_context, options)
 
     if options.provenance:
         research_obj = cwltool.cwlprov.ro.ResearchObject(
-            temp_prefix_ro=options.tmp_outdir_prefix,
+            temp_prefix_ro=tmp_outdir_prefix,
             orcid=options.orcid,
             full_name=options.cwl_full_name,
             fsaccess=runtime_context.make_fs_access(""),
@@ -3701,7 +3703,8 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
             loading_context, uri = cwltool.load_tool.resolve_and_validate_document(
                 loading_context, workflowobj, uri
             )
-            assert loading_context.loader
+            if not loading_context.loader:
+                raise RuntimeError("cwltool loader is not set.")
             processobj, metadata = loading_context.loader.resolve_ref(uri)
             processobj = cast(Union[CommentedMap, CommentedSeq], processobj)
 
@@ -3748,10 +3751,8 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                     )
                 raise
 
-            # We make a ToilFSAccess to access URLs with, but it has no
-            # FileStore so it can't do toildir: and toilfile:
-            fs_access = ToilFsAccess(options.basedir)
-            fill_in_defaults(tool.tool["inputs"], initialized_job_order, fs_access)
+            # Leave the defaults un-filled in the top-level order. The tool or
+            # workflow will fill them when it runs
 
             for inp in tool.tool["inputs"]:
                 if (
@@ -3809,6 +3810,8 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
 
             # Import all the input files, some of which may be missing optional
             # files.
+            logger.info("Importing input files...")
+            fs_access = ToilFsAccess(options.basedir)
             import_files(
                 file_import_function,
                 fs_access,
@@ -3816,11 +3819,14 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                 existing,
                 initialized_job_order,
                 skip_broken=True,
+                skip_remote=options.reference_inputs,
                 bypass_file_store=options.bypass_file_store,
+                log_level=logging.INFO,
             )
             # Import all the files associated with tools (binaries, etc.).
             # Not sure why you would have an optional secondary file here, but
             # the spec probably needs us to support them.
+            logger.info("Importing tool-associated files...")
             visitSteps(
                 tool,
                 functools.partial(
@@ -3830,7 +3836,9 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                     fileindex,
                     existing,
                     skip_broken=True,
+                    skip_remote=options.reference_inputs,
                     bypass_file_store=options.bypass_file_store,
+                    log_level=logging.INFO,
                 ),
             )
 
@@ -3843,7 +3851,8 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                 # were required.
                 rm_unprocessed_secondary_files(param_value)
 
-            logger.debug("tool %s", tool)
+            logger.info("Creating root job")
+            logger.debug("Root tool: %s", tool)
             try:
                 wf1, _ = makeJob(
                     tool=tool,
@@ -3856,6 +3865,7 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                 logging.error(err)
                 return CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE
             wf1.cwljob = initialized_job_order
+            logger.info("Starting workflow")
             try:
                 outobj = toil.start(wf1)
             except FailedJobsException as err:
@@ -3871,13 +3881,20 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
 
         # Now the workflow has completed. We need to make sure the outputs (and
         # inputs) end up where the user wants them to be.
-
+        logger.info("Collecting workflow outputs...")
         outobj = resolve_dict_w_promises(outobj)
 
         # Stage files. Specify destination bucket if specified in CLI
         # options. If destination bucket not passed in,
         # options.destBucket's value will be None.
-        toilStageFiles(toil, outobj, outdir, destBucket=options.destBucket)
+        toilStageFiles(
+            toil,
+            outobj,
+            outdir,
+            destBucket=options.destBucket,
+            log_level=logging.INFO
+        )
+        logger.info("Stored workflow outputs")
 
         if runtime_context.research_obj is not None:
             cwltool.cwlprov.writablebagfile.create_job(
@@ -3904,7 +3921,8 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
                 ("File",),
                 functools.partial(add_sizes, runtime_context.make_fs_access("")),
             )
-            assert document_loader
+            if not document_loader:
+                raise RuntimeError("cwltool loader is not set.")
             prov_dependencies = cwltool.main.prov_deps(
                 workflowobj, document_loader, uri
             )
@@ -3914,6 +3932,7 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
             )
 
         if not options.destBucket and options.compute_checksum:
+            logger.info("Computing output file checksums...")
             visit_class(
                 outobj,
                 ("File",),
@@ -3922,12 +3941,14 @@ def main(args: Optional[List[str]] = None, stdout: TextIO = sys.stdout) -> int:
 
         visit_class(outobj, ("File",), MutationManager().unset_generation)
         stdout.write(json.dumps(outobj, indent=4, default=str))
+        stdout.write("\n")
+        logger.info("CWL run complete!")
 
     return 0
 
 
 def find_default_container(
-    args: argparse.Namespace, builder: cwltool.builder.Builder
+    args: Namespace, builder: cwltool.builder.Builder
 ) -> Optional[str]:
     """Find the default constructor by consulting a Toil.options object."""
     if args.default_container: