toil 6.1.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

toil/wdl/wdltoil.py

@@ -24,6 +24,7 @@ import shutil
 import stat
 import subprocess
 import sys
+import textwrap
 import uuid
 from contextlib import ExitStack, contextmanager
 from graphlib import TopologicalSorter
@@ -47,14 +48,15 @@ from urllib.parse import quote, unquote, urljoin, urlsplit
 
 import WDL.Error
 import WDL.runtime.config
-from configargparse import ArgParser, SUPPRESS
+from configargparse import ArgParser
 from WDL._util import byte_size_units, strip_leading_whitespace
 from WDL.CLI import print_error
 from WDL.runtime.backend.docker_swarm import SwarmContainer
 from WDL.runtime.backend.singularity import SingularityContainer
 from WDL.runtime.task_container import TaskContainer
 
-from toil.common import Toil, addOptions, check_and_create_default_config_file
+from toil.batchSystems.abstractBatchSystem import InsufficientSystemResources
+from toil.common import Toil, addOptions
 from toil.fileStores import FileID
 from toil.fileStores.abstractFileStore import AbstractFileStore
 from toil.job import (AcceleratorRequirement,
@@ -62,17 +64,19 @@ from toil.job import (AcceleratorRequirement,
                       Promise,
                       Promised,
                       TemporaryID,
-                      accelerators_fully_satisfy,
                       parse_accelerator,
                       unwrap,
                       unwrap_all)
-from toil.jobStores.abstractJobStore import (AbstractJobStore,
-                                             UnimplementedURLException)
-from toil.lib.conversions import convert_units, human2bytes
+from toil.jobStores.abstractJobStore import (AbstractJobStore, UnimplementedURLException,
+                                             InvalidImportExportUrlException, LocatorException)
+from toil.lib.conversions import convert_units, human2bytes, strtobool
 from toil.lib.io import mkdtemp
 from toil.lib.memoize import memoize
 from toil.lib.misc import get_user_name
+from toil.lib.resources import ResourceMonitor
 from toil.lib.threading import global_mutex
+from toil.provisioners.clusterScaler import JobTooBigError
+
 
 logger = logging.getLogger(__name__)
 
@@ -86,19 +90,30 @@ def wdl_error_reporter(task: str, exit: bool = False, log: Callable[[str], None]
     try:
         yield
     except (
+        WDL.Error.EvalError,
         WDL.Error.SyntaxError,
         WDL.Error.ImportError,
         WDL.Error.ValidationError,
         WDL.Error.MultipleValidationErrors,
-        FileNotFoundError
+        FileNotFoundError,
+        InsufficientSystemResources,
+        LocatorException,
+        InvalidImportExportUrlException,
+        UnimplementedURLException,
+        JobTooBigError
     ) as e:
-        log("Could not " + task)
+        # Don't expose tracebacks to the user for exceptions that may be expected
+        log("Could not " + task + " because:")
+        
         # These are the errors that MiniWDL's parser can raise and its reporter
-        # can report. See
+        # can report (plus some extras). See
         # https://github.com/chanzuckerberg/miniwdl/blob/a780b1bf2db61f18de37616068968b2bb4c2d21c/WDL/CLI.py#L91-L97.
         #
         # We are going to use MiniWDL's pretty printer to print them.
+        # Make the MiniWDL stuff on stderr loud so people see it
+        sys.stderr.write("\n" + "🚨" * 3 + "\n")
         print_error(e)
+        sys.stderr.write("🚨" * 3 + "\n\n")
         if exit:
             # Stop right now
             sys.exit(1)
@@ -524,6 +539,8 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
     def __init__(self, file_store: AbstractFileStore, execution_dir: Optional[str] = None):
         """
         Set up the standard library.
+
+        :param execution_dir: Directory to use as the working directory for workflow code.
         """
         # TODO: Just always be the 1.2 standard library.
         wdl_version = "1.2"
@@ -542,27 +559,67 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
         # UUID to differentiate which node files are virtualized from
         self._parent_dir_to_ids: Dict[str, uuid.UUID] = dict()
 
+        # Map forward from virtualized files to absolute devirtualized ones.
+        self._virtualized_to_devirtualized: Dict[str, str] = {}
+        # Allow mapping back from absolute devirtualized files to virtualized
+        # paths, to save re-uploads.
+        self._devirtualized_to_virtualized: Dict[str, str] = {}
+
         self._execution_dir = execution_dir
 
+    def share_files(self, other: "ToilWDLStdLibBase") -> None:
+        """
+        Share caches for devirtualizing and virtualizing files with another instance.
+
+        Files devirtualized by one instance can be re-virtualized back to their
+        original virtualized filenames by the other.
+        """
+
+        if id(self._virtualized_to_devirtualized) != id(other._virtualized_to_devirtualized):
+            # Merge the virtualized to devirtualized mappings
+            self._virtualized_to_devirtualized.update(other._virtualized_to_devirtualized)
+            other._virtualized_to_devirtualized = self._virtualized_to_devirtualized
+
+        if id(self._devirtualized_to_virtualized) != id(other._devirtualized_to_virtualized):
+            # Merge the devirtualized to virtualized mappings
+            self._devirtualized_to_virtualized.update(other._devirtualized_to_virtualized)
+            other._devirtualized_to_virtualized = self._devirtualized_to_virtualized
+
     @memoize
     def _devirtualize_filename(self, filename: str) -> str:
         """
         'devirtualize' filename passed to a read_* function: return a filename that can be open()ed
         on the local host.
         """
-
-        return self.devirtualze_to(filename, self._file_store.localTempDir, self._file_store, self._execution_dir) 
+        
+        result = self.devirtualize_to(filename, self._file_store.localTempDir, self._file_store, self._execution_dir)
+        # Store the back mapping
+        self._devirtualized_to_virtualized[result] = filename
+        # And the forward
+        self._virtualized_to_devirtualized[filename] = result
+        return result
 
     @staticmethod
-    def devirtualze_to(filename: str, dest_dir: str, file_source: Union[AbstractFileStore, Toil], execution_dir: Optional[str]) -> str:
+    def devirtualize_to(filename: str, dest_dir: str, file_source: Union[AbstractFileStore, Toil], execution_dir: Optional[str]) -> str:
         """
         Download or export a WDL virtualized filename/URL to the given directory.
 
-        Makes sure sibling files stay siblings and files with the same name don't clobber each other. Called from within this class for tasks, and statically at the end of the workflow for outputs.
+        The destination directory must already exist.
+
+        Makes sure sibling files stay siblings and files with the same name
+        don't clobber each other. Called from within this class for tasks, and
+        statically at the end of the workflow for outputs.
 
-        Returns the local path to the file.
+        Returns the local path to the file. If it already had a local path
+        elsewhere, it might not actually be put in dest_dir.
         """
 
+        if not os.path.isdir(dest_dir):
+            # os.mkdir fails saying the directory *being made* caused a
+            # FileNotFoundError. So check the dest_dir before trying to make
+            # directories under it.
+            raise RuntimeError(f"Cannot devirtualize {filename} into nonexistent directory {dest_dir}")
+
         # TODO: Support people doing path operations (join, split, get parent directory) on the virtualized filenames.
         # TODO: For task inputs, we are supposed to make sure to put things in the same directory if they came from the same directory. See <https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md#task-input-localization>
         if is_url(filename):
@@ -597,8 +654,12 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             if filename.startswith(TOIL_URI_SCHEME):
                 # Get a local path to the file
                 if isinstance(file_source, AbstractFileStore):
-                    # Read from the file store
-                    result = file_source.readGlobalFile(file_id, dest_path)
+                    # Read from the file store.
+                    # File is not allowed to be modified by the task. See
+                    # <https://github.com/openwdl/wdl/issues/495>.
+                    # We try to get away with symlinks and hope the task
+                    # container can mount the destination file.
+                    result = file_source.readGlobalFile(file_id, dest_path, mutable=False, symlink=True)
                 elif isinstance(file_source, Toil):
                     # Read from the Toil context
                     file_source.export_file(file_id, dest_path)
@@ -628,6 +689,7 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             raise RuntimeError(f"Virtualized file {filename} looks like a local file but isn't!")
         return result
 
+    @memoize
     def _virtualize_filename(self, filename: str) -> str:
         """
         from a local path in write_dir, 'virtualize' into the filename as it should present in a
@@ -636,21 +698,36 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
 
         if is_url(filename):
             # Already virtual
-            logger.debug('Already virtualized %s as WDL file %s', filename, filename)
+            logger.debug('Already virtual: %s', filename)
             return filename
 
         # Otherwise this is a local file and we want to fake it as a Toil file store file
 
-        # To support relative paths from execution directory, join the execution dir and filename
-        # If filename is already an abs path, join() will not do anything
+        # Make it an absolute path
         if self._execution_dir is not None:
-            file_id = self._file_store.writeGlobalFile(os.path.join(self._execution_dir, filename))
+            # To support relative paths from execution directory, join the execution dir and filename
+            # If filename is already an abs path, join() will not do anything
+            abs_filename = os.path.join(self._execution_dir, filename)
         else:
-            file_id = self._file_store.writeGlobalFile(filename)
-        dir = os.path.dirname(os.path.abspath(filename)) # is filename always an abspath?
-        parent_id = self._parent_dir_to_ids.setdefault(dir, uuid.uuid4())
-        result = pack_toil_uri(file_id, parent_id, os.path.basename(filename))
+            abs_filename = os.path.abspath(filename)
+
+        if abs_filename in self._devirtualized_to_virtualized:
+            # This is a previously devirtualized thing so we can just use the
+            # virtual version we remembered instead of reuploading it.
+            result = self._devirtualized_to_virtualized[abs_filename]
+            logger.debug("Re-using virtualized WDL file %s for %s", result, filename)
+            return result
+
+        file_id = self._file_store.writeGlobalFile(abs_filename)
+        
+        file_dir = os.path.dirname(abs_filename)
+        parent_id = self._parent_dir_to_ids.setdefault(file_dir, uuid.uuid4())
+        result = pack_toil_uri(file_id, parent_id, os.path.basename(abs_filename))
         logger.debug('Virtualized %s as WDL file %s', filename, result)
+        # Remember the upload in case we share a cache
+        self._devirtualized_to_virtualized[abs_filename] = result
+        # And remember the local path in case we want a redownload
+        self._virtualized_to_devirtualized[result] = abs_filename
         return result
 
 class ToilWDLStdLibTaskCommand(ToilWDLStdLibBase):
@@ -695,7 +772,7 @@ class ToilWDLStdLibTaskCommand(ToilWDLStdLibBase):
         logger.debug('Devirtualized %s as out-of-container file %s', filename, result)
         return result
 
-
+    @memoize
     def _virtualize_filename(self, filename: str) -> str:
         """
         From a local path in write_dir, 'virtualize' into the filename as it should present in a
@@ -717,10 +794,11 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
     functions only allowed in task output sections.
     """
 
-    def __init__(self, file_store: AbstractFileStore, stdout_path: str, stderr_path: str, current_directory_override: Optional[str] = None):
+    def __init__(self, file_store: AbstractFileStore, stdout_path: str, stderr_path: str, file_to_mountpoint: Dict[str, str], current_directory_override: Optional[str] = None):
         """
         Set up the standard library for a task output section. Needs to know
-        where standard output and error from the task have been stored.
+        where standard output and error from the task have been stored, and
+        what local paths to pretend are where for resolving symlinks.
 
         If current_directory_override is set, resolves relative paths and globs
         from there instead of from the real current directory.
@@ -738,6 +816,9 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
         self._stdout_used = False
         self._stderr_used = False
 
+        # Reverse and store the file mount dict
+        self._mountpoint_to_file = {v: k for k, v in file_to_mountpoint.items()}
+
         # Remember current directory
         self._current_directory_override = current_directory_override
 
@@ -806,7 +887,7 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
         work_dir = '.' if not self._current_directory_override else self._current_directory_override
 
         # TODO: get this to run in the right container if there is one
-        # Bash (now?) has a compgen builtin for shell completion that can evaluate a glob where the glob is in a quotes string that might have spaces in it. See <https://unix.stackexchange.com/a/616608>.
+        # Bash (now?) has a compgen builtin for shell completion that can evaluate a glob where the glob is in a quoted string that might have spaces in it. See <https://unix.stackexchange.com/a/616608>.
         # This will handle everything except newlines in the filenames.
         # TODO: Newlines in the filenames?
         # Since compgen will return 1 if nothing matches, we need to allow a failing exit code here.
@@ -844,6 +925,7 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
 
         return super()._devirtualize_filename(filename)
 
+    @memoize
     def _virtualize_filename(self, filename: str) -> str:
         """
         Go from a local disk filename to a virtualized WDL-side filename.
@@ -854,11 +936,46 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
         """
 
         if not is_url(filename) and not filename.startswith('/'):
-            # We are getting a bare relative path the supposedly devirtualized side.
+            # We are getting a bare relative path on the supposedly devirtualized side.
             # Find a real path to it relative to the current directory override.
             work_dir = '.' if not self._current_directory_override else self._current_directory_override
             filename = os.path.join(work_dir, filename)
 
+        if filename in self._devirtualized_to_virtualized:
+            result = self._devirtualized_to_virtualized[filename]
+            logger.debug("Re-using virtualized filename %s for %s", result, filename)
+            return result
+
+        if os.path.islink(filename):
+            # Recursively resolve symlinks
+            here = filename
+            # Notice if we have a symlink loop
+            seen = {here}
+            while os.path.islink(here):
+                dest = os.readlink(here)
+                if not dest.startswith('/'):
+                    # Make it absolute
+                    dest = os.path.join(os.path.dirname(here), dest)
+                here = dest
+                if here in self._mountpoint_to_file:
+                    # This points to something mounted into the container, so use that path instead.
+                    here = self._mountpoint_to_file[here]
+                if here in self._devirtualized_to_virtualized:
+                    # Check the virtualized filenames before following symlinks
+                    # all the way back to workflow inputs.
+                    result = self._devirtualized_to_virtualized[here]
+                    logger.debug("Re-using virtualized filename %s for %s linked from %s", result, here, filename)
+                    return result
+                if here in seen:
+                    raise RuntimeError(f"Symlink {filename} leads to symlink loop at {here}")
+                seen.add(here)
+
+            if os.path.exists(here):
+                logger.debug("Handling symlink %s ultimately to %s", filename, here)
+            else:
+                logger.error("Handling broken symlink %s ultimately to %s", filename, here)
+            filename = here
+
         return super()._virtualize_filename(filename)
 
 def evaluate_named_expression(context: Union[WDL.Error.SourceNode, WDL.Error.SourcePosition], name: str, expected_type: Optional[WDL.Type.Base], expression: Optional[WDL.Expr.Base], environment: WDLBindings, stdlib: WDL.StdLib.Base) -> WDL.Value.Base:
@@ -1110,8 +1227,10 @@ def drop_missing_files(environment: WDLBindings, current_directory_override: Opt
                 logger.warning('File %s with type %s does not actually exist at its URI', filename, value_type)
                 return None
         else:
+            # Get the absolute path, not resolving symlinks
             effective_path = os.path.abspath(os.path.join(work_dir, filename))
-            if os.path.exists(effective_path):
+            if os.path.islink(effective_path) or os.path.exists(effective_path):
+                # This is a broken symlink or a working symlink or a file.
                 return filename
             else:
                 logger.warning('File %s with type %s does not actually exist at %s', filename, value_type, effective_path)
@@ -1125,7 +1244,7 @@ def get_file_paths_in_bindings(environment: WDLBindings) -> List[str]:
     duplicates are removed.
 
     TODO: Duplicative with WDL.runtime.task._fspaths, except that is internal
-    and supports Direcotry objects.
+    and supports Directory objects.
     """
 
     paths = []
@@ -1250,7 +1369,7 @@ class WDLBaseJob(Job):
         # may have coalesced postprocessing steps deferred by several levels of
         # jobs returning other jobs' promised RVs.
         self._postprocessing_steps: List[Tuple[str, Union[str, Promised[WDLBindings]]]] = []
-        
+
         self._wdl_options = wdl_options if wdl_options is not None else {}
 
         assert self._wdl_options.get("container") is not None
@@ -1306,7 +1425,7 @@ class WDLBaseJob(Job):
 
         for action, argument in self._postprocessing_steps:
 
-            logger.debug("Apply postprocessing setp: (%s, %s)", action, argument)
+            logger.debug("Apply postprocessing step: (%s, %s)", action, argument)
 
             # Interpret the mini language of postprocessing steps.
             # These are too small to justify being their own separate jobs.
@@ -1378,7 +1497,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
         self._namespace = namespace
         self._task_path = task_path
 
-    @report_wdl_errors("evaluate task code")
+    @report_wdl_errors("evaluate task code", exit=True)
     def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
         """
         Evaluate inputs and runtime and schedule the task.
@@ -1407,7 +1526,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
         # Evaluate the runtime section
         runtime_bindings = evaluate_call_inputs(self._task, self._task.runtime, bindings, standard_library)
 
-        # Fill these in with not-None if the workflow asks for each resource. 
+        # Fill these in with not-None if the workflow asks for each resource.
         runtime_memory: Optional[int] = None
         runtime_cores: Optional[float] = None
         runtime_disk: Optional[int] = None
@@ -1529,6 +1648,123 @@ class WDLTaskJob(WDLBaseJob):
         self._namespace = namespace
         self._task_path = task_path
 
+    ###
+    # Runtime code injection system
+    ###
+
+    # WDL runtime code injected in the container communicates back to the rest
+    # of the runtime through files in this directory.
+    INJECTED_MESSAGE_DIR = ".toil_wdl_runtime"
+
+    def add_injections(self, command_string: str, task_container: TaskContainer) -> str:
+        """
+        Inject extra Bash code from the Toil WDL runtime into the command for the container.
+
+        Currently doesn't implement the MiniWDL plugin system, but does add
+        resource usage monitoring to Docker containers.
+        """
+        if isinstance(task_container, SwarmContainer):
+            # We're running on Docker Swarm, so we need to monitor CPU usage
+            # and so on from inside the container, since it won't be attributed
+            # to Toil child processes in the leader's self-monitoring.
+            # TODO: Mount this from a file Toil installs instead or something.
+            script = textwrap.dedent("""\
+                function _toil_resource_monitor () {
+                    # Turn off error checking and echo in here
+                    set +ex
+                    MESSAGE_DIR="${1}"
+                    mkdir -p "${MESSAGE_DIR}"
+
+                    function sample_cpu_usec() {
+                        if [[ -f  /sys/fs/cgroup/cpu.stat ]] ; then
+                            awk '{ if ($1 == "usage_usec") {print $2} }' /sys/fs/cgroup/cpu.stat
+                        elif [[ -f /sys/fs/cgroup/cpuacct/cpuacct.stat ]] ; then
+                            echo $(( $(head -n 1 /sys/fs/cgroup/cpuacct/cpuacct.stat | cut -f2 -d' ') * 10000 ))
+                        fi
+                    }
+
+                    function sample_memory_bytes() {
+                        if [[ -f /sys/fs/cgroup/memory.stat ]] ; then
+                            awk '{ if ($1 == "anon") { print $2 } }' /sys/fs/cgroup/memory.stat
+                        elif [[ -f /sys/fs/cgroup/memory/memory.stat ]] ; then
+                            awk '{ if ($1 == "total_rss") { print $2 } }' /sys/fs/cgroup/memory/memory.stat
+                        fi
+                    }
+
+                    while true ; do
+                        printf "CPU\\t" >> ${MESSAGE_DIR}/resources.tsv
+                        sample_cpu_usec >> ${MESSAGE_DIR}/resources.tsv
+                        printf "Memory\\t" >> ${MESSAGE_DIR}/resources.tsv
+                        sample_memory_bytes >> ${MESSAGE_DIR}/resources.tsv
+                        sleep 1
+                    done
+                }
+                """)
+            parts = [script, f"_toil_resource_monitor {self.INJECTED_MESSAGE_DIR} &", command_string]
+            return "\n".join(parts)
+        else:
+            return command_string
+
+    def handle_injection_messages(self, outputs_library: ToilWDLStdLibTaskOutputs) -> None:
+        """
+        Handle any data received from injected runtime code in the container.
+        """
+
+        message_files = outputs_library._glob(WDL.Value.String(os.path.join(self.INJECTED_MESSAGE_DIR, "*")))
+        logger.debug("Handling message files: %s", message_files)
+        for message_file in message_files.value:
+            self.handle_message_file(message_file.value)
+
+    def handle_message_file(self, file_path: str) -> None:
+        """
+        Handle a message file received from in-container injected code.
+
+        Takes the host-side path of the file.
+        """
+        if os.path.basename(file_path) == "resources.tsv":
+            # This is a TSV of resource usage info.
+            first_cpu_usec: Optional[int] = None
+            last_cpu_usec: Optional[int] = None
+            max_memory_bytes: Optional[int] = None
+
+            for line in open(file_path):
+                if not line.endswith("\n"):
+                    # Skip partial lines
+                    continue
+                # For each full line we got
+                parts = line.strip().split("\t")
+                if len(parts) != 2:
+                    # Skip odd-shaped lines
+                    continue
+                if parts[0] == "CPU":
+                    # Parse CPU usage
+                    cpu_usec = int(parts[1])
+                    # Update summary stats
+                    if first_cpu_usec is None:
+                        first_cpu_usec = cpu_usec
+                    last_cpu_usec = cpu_usec
+                elif parts[0] == "Memory":
+                    # Parse memory usage
+                    memory_bytes = int(parts[1])
+                    # Update summary stats
+                    if max_memory_bytes is None or max_memory_bytes < memory_bytes:
+                        max_memory_bytes = memory_bytes
+
+            if max_memory_bytes is not None:
+                logger.info("Container used at about %s bytes of memory at peak", max_memory_bytes)
+                # Treat it as if used by a child process
+                ResourceMonitor.record_extra_memory(max_memory_bytes // 1024)
+            if last_cpu_usec is not None:
+                assert(first_cpu_usec is not None)
+                cpu_seconds = (last_cpu_usec - first_cpu_usec) / 1000000
+                logger.info("Container used about %s seconds of CPU time", cpu_seconds)
+                # Treat it as if used by a child process
+                ResourceMonitor.record_extra_cpu(cpu_seconds)
+
+    ###
+    # Helper functions to work out what containers runtime we can use
+    ###
+
     def can_fake_root(self) -> bool:
         """
         Determine if --fakeroot is likely to work for Singularity.
@@ -1559,7 +1795,7 @@ class WDLTaskJob(WDLBaseJob):
         """
         return "KUBERNETES_SERVICE_HOST" not in os.environ
 
-    @report_wdl_errors("run task command")
+    @report_wdl_errors("run task command", exit=True)
     def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
         """
         Actually run the task.
@@ -1575,7 +1811,7 @@ class WDLTaskJob(WDLBaseJob):
         bindings = unwrap(self._task_internal_bindings)
         # And the bindings from evaluating the runtime section
         runtime_bindings = unwrap(self._runtime_bindings)
-        
+
         # We have all the resources we need, so run the task
 
         if shutil.which('singularity') and self._wdl_options.get("container") in ["singularity", "auto"]:
@@ -1644,9 +1880,20 @@ class WDLTaskJob(WDLBaseJob):
         workdir_in_container: Optional[str] = None
 
         if self._task.command:
-            # When the command string references a File, we need to get a path to the file on a local disk, which the commnad will be able to actually use, accounting for e.g. containers.
-            # TODO: Figure out whan the command template actually uses File values and lazily download them.
-            # For now we just grab all the File values in the inside-the-task environment, since any of them *might* be used.
+            # When the command string references a File, we need to get a path
+            # to the file on a local disk, which the commnad will be able to
+            # actually use, accounting for e.g. containers.
+            #
+            # TODO: Figure out whan the command template actually uses File
+            # values and lazily download them.
+            #
+            # For now we just grab all the File values in the inside-the-task
+            # environment, since any of them *might* be used.
+            #
+            # Some also might be expected to be adjacent to files that are
+            # used, like a BAI that doesn't get referenced in a command line
+            # but must be next to its BAM.
+            #
             # TODO: MiniWDL can parallelize the fetch
             bindings = devirtualize_files(bindings, standard_library)
 
@@ -1721,6 +1968,7 @@ class WDLTaskJob(WDLBaseJob):
             # them all new paths in task_container.input_path_map which we can
             # read. We also get a task_container.host_path() to go the other way.
             add_paths(task_container, get_file_paths_in_bindings(bindings))
+            # This maps from oustide container to inside container
             logger.debug("Using container path map: %s", task_container.input_path_map)
 
             # Replace everything with in-container paths for the command.
@@ -1767,6 +2015,9 @@ class WDLTaskJob(WDLBaseJob):
             # Work out the command string, and unwrap it
             command_string: str = hacky_dedent(evaluate_named_expression(self._task, "command", WDL.Type.String(), self._task.command, contained_bindings, command_library).coerce(WDL.Type.String()).value)
 
+            # Do any command injection we might need to do
+            command_string = self.add_injections(command_string, task_container)
+
             # Grab the standard out and error paths. MyPy complains if we call
             # them because in the current MiniWDL version they are untyped.
             # TODO: MyPy will complain if we accomodate this and they later
@@ -1786,8 +2037,16 @@ class WDLTaskJob(WDLBaseJob):
                         with ExitStack() as cleanup:
                             task_container._pull(miniwdl_logger, cleanup)
 
-            # Run the command in the container
+            # Log that we are about to run the command in the container
             logger.info('Executing command in %s: %s', task_container, command_string)
+
+            # Now our inputs are all downloaded. Let debugging break in (after command is logged).
+            # But we need to hint which host paths are meant to be which container paths
+            host_and_job_paths: List[Tuple[str, str]] = [(k, v) for k, v in task_container.input_path_map.items()]
+            self.files_downloaded_hook(host_and_job_paths)
+
+            # TODO: Really we might want to set up a fake container working directory, to actually help the user.
+
             try:
                 task_container.run(miniwdl_logger, command_string)
             except Exception:
@@ -1834,15 +2093,14 @@ class WDLTaskJob(WDLBaseJob):
         # container-determined strings that are absolute paths to WDL File
         # objects, and like MiniWDL we can say we only support
         # working-directory-based relative paths for globs.
-        outputs_library = ToilWDLStdLibTaskOutputs(file_store, host_stdout_txt, host_stderr_txt, current_directory_override=workdir_in_container)
+        outputs_library = ToilWDLStdLibTaskOutputs(file_store, host_stdout_txt, host_stderr_txt, task_container.input_path_map, current_directory_override=workdir_in_container)
+        # Make sure files downloaded as inputs get re-used if we re-upload them.
+        outputs_library.share_files(standard_library)
         output_bindings = evaluate_output_decls(self._task.outputs, bindings, outputs_library)
 
         # Now we know if the standard output and error were sent somewhere by
         # the workflow. If not, we should report them to the leader.
 
-        # Drop any files from the output which don't actually exist
-        output_bindings = drop_missing_files(output_bindings, current_directory_override=workdir_in_container)
-
         if not outputs_library.stderr_used() and os.path.exists(host_stderr_txt):
             size = os.path.getsize(host_stderr_txt)
             logger.info('Unused standard error at %s of %d bytes', host_stderr_txt, size)
@@ -1857,10 +2115,17 @@ class WDLTaskJob(WDLBaseJob):
                 # Save the whole output stream because the workflow didn't capture it.
                 file_store.log_user_stream(self._task_path + '.stdout', open(host_stdout_txt, 'rb'))
 
-        # TODO: Check the output bindings against the types of the decls so we
-        # can tell if we have a null in a value that is supposed to not be
-        # nullable. We can't just look at the types on the values themselves
-        # because those are all the non-nullable versions.
+        # Collect output messages from any code Toil injected into the task.
+        self.handle_injection_messages(outputs_library)
+
+        # Drop any files from the output which don't actually exist
+        output_bindings = drop_missing_files(output_bindings, current_directory_override=workdir_in_container)
+        for decl in self._task.outputs:
+            if not decl.type.optional and output_bindings[decl.name].value is None:
+                    # We have an unacceptable null value. This can happen if a file
+                    # is missing but not optional. Don't let it out to annoy the
+                    # next task.
+                    raise WDL.Error.EvalError(decl, f"non-optional value {decl.name} = {decl.expr} is missing")
 
         # Upload any files in the outputs if not uploaded already. Accounts for how relative paths may still need to be container-relative.
         output_bindings = virtualize_files(output_bindings, outputs_library)
@@ -2893,13 +3158,15 @@ def main() -> None:
             raise RuntimeError("The output of the WDL job is not a binding.")
 
         # Fetch all the output files
-        # TODO: deduplicate with _devirtualize_filename
         def devirtualize_output(filename: str) -> str:
             """
             'devirtualize' a file using the "toil" object instead of a filestore.
             Returns its local path.
             """
-            return ToilWDLStdLibBase.devirtualze_to(filename, output_directory, toil, execution_dir) 
+            # Make sure the output directory exists if we have output files
+            # that might need to use it.
+            os.makedirs(output_directory, exist_ok=True)
+            return ToilWDLStdLibBase.devirtualize_to(filename, output_directory, toil, execution_dir)
 
         # Make all the files local files
         output_bindings = map_over_files_in_bindings(output_bindings, devirtualize_output)