toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl

toil/wdl/wdltoil.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 import asyncio
+import copy
 import errno
 import hashlib
 import io
@@ -62,13 +63,14 @@ else:
 
 from functools import partial
 from urllib.error import HTTPError
-from urllib.parse import quote, unquote, urljoin, urlsplit, urlparse
+from urllib.parse import quote, unquote, urljoin, urlsplit
 
 import WDL.Error
 import WDL.runtime.config
 from configargparse import ArgParser, Namespace
 from WDL._util import byte_size_units, chmod_R_plus
-from WDL.CLI import print_error
+from WDL.CLI import print_error, outline
+import WDL.Lint
 from WDL.runtime.backend.docker_swarm import SwarmContainer
 from WDL.runtime.backend.singularity import SingularityContainer
 from WDL.runtime.error import DownloadFailed
@@ -110,6 +112,7 @@ from toil.lib.misc import get_user_name
 from toil.lib.resources import ResourceMonitor
 from toil.lib.threading import global_mutex
 from toil.provisioners.clusterScaler import JobTooBigError
+from toil.lib.url import URLAccess
 
 logger = logging.getLogger(__name__)
 
@@ -292,207 +295,6 @@ def report_wdl_errors(
     return decorator
 
 
-def remove_common_leading_whitespace(
-    expression: WDL.Expr.String,
-    tolerate_blanks: bool = True,
-    tolerate_dedents: bool = False,
-    tolerate_all_whitespace: bool = True,
-    debug: bool = False,
-) -> WDL.Expr.String:
-    """
-    Remove "common leading whitespace" as defined in the WDL 1.1 spec.
-
-    See <https://github.com/openwdl/wdl/blob/main/versions/1.1/SPEC.md#stripping-leading-whitespace>.
-
-    Operates on a WDL.Expr.String expression that has already been parsed.
-
-    :param tolerate_blanks: If True, don't allow totally blank lines to zero
-        the common whitespace.
-
-    :param tolerate_dedents: If True, remove as much of the whitespace on the
-        first indented line as is found on subesquent lines, regardless of
-        whether later lines are out-dented relative to it.
-
-    :param tolerate_all_whitespace: If True, don't allow all-whitespace lines
-        to reduce the common whitespace prefix.
-
-    :param debug: If True, the function will show its work by logging at debug
-        level.
-    """
-
-    # The expression has a "parts" list consisting of interleaved string
-    # literals and placeholder expressions.
-    #
-    # TODO: We assume that there are no newlines in the placeholders.
-    #
-    # TODO: Look at the placeholders and their line and end_line values and try
-    # and guess if they should reduce the amount of common whitespace.
-
-    if debug:
-        logger.debug("Parts: %s", expression.parts)
-
-    # We split the parts list into lines, which are also interleaved string
-    # literals and placeholder expressions.
-    lines: list[list[str | WDL.Expr.Placeholder]] = [[]]
-    for part in expression.parts:
-        if isinstance(part, str):
-            # It's a string. Split it into lines.
-            part_lines = part.split("\n")
-            # Part before any newline goes at the end of the current line
-            lines[-1].append(part_lines[0])
-            for part_line in part_lines[1:]:
-                # Any part after a newline starts a new line
-                lines.append([part_line])
-        else:
-            # It's a placeholder. Put it at the end of the current line.
-            lines[-1].append(part)
-
-    if debug:
-        logger.debug("Lines: %s", lines)
-
-    # Then we compute the common amount of leading whitespace on all the lines,
-    # looking at the first string literal.
-    # This will be the longest common whitespace prefix, or None if not yet detected.
-    common_whitespace_prefix: str | None = None
-    for line in lines:
-        if len(line) == 0:
-            # TODO: how should totally empty lines be handled? Not in the spec!
-            if not tolerate_blanks:
-                # There's no leading whitespace here!
-                common_whitespace_prefix = ""
-            continue
-        elif isinstance(line[0], WDL.Expr.Placeholder):
-            # TODO: How can we convert MiniWDL's column numbers into space/tab counts or sequences?
-            #
-            # For now just skip these too.
-            continue
-        else:
-            # The line starts with a string
-            assert isinstance(line[0], str)
-            if len(line[0]) == 0:
-                # Still totally empty though!
-                if not tolerate_blanks:
-                    # There's no leading whitespace here!
-                    common_whitespace_prefix = ""
-                continue
-            if (
-                len(line) == 1
-                and tolerate_all_whitespace
-                and all(x in (" ", "\t") for x in line[0])
-            ):
-                # All-whitespace lines shouldn't count
-                continue
-            # TODO: There are good algorithms for common prefixes. This is a bad one.
-            # Find the number of leading whitespace characters
-            line_whitespace_end = 0
-            while line_whitespace_end < len(line[0]) and line[0][
-                line_whitespace_end
-            ] in (" ", "\t"):
-                line_whitespace_end += 1
-            # Find the string of leading whitespace characters
-            line_whitespace_prefix = line[0][:line_whitespace_end]
-
-            if " " in line_whitespace_prefix and "\t" in line_whitespace_prefix:
-                # Warn and don't change anything if spaces and tabs are mixed, per the spec.
-                logger.warning(
-                    "Line in command at %s mixes leading spaces and tabs! Not removing leading whitespace!",
-                    expression.pos,
-                )
-                return expression
-
-            if common_whitespace_prefix is None:
-                # This is the first line we found, so it automatically has the common prefic
-                common_whitespace_prefix = line_whitespace_prefix
-            elif not tolerate_dedents:
-                # Trim the common prefix down to what we have for this line
-                if not line_whitespace_prefix.startswith(common_whitespace_prefix):
-                    # Shorten to the real shared prefix.
-                    # Hackily make os.path do it for us,
-                    # character-by-character. See
-                    # <https://stackoverflow.com/a/6718435>
-                    common_whitespace_prefix = os.path.commonprefix(
-                        [common_whitespace_prefix, line_whitespace_prefix]
-                    )
-
-    if common_whitespace_prefix is None:
-        common_whitespace_prefix = ""
-
-    if debug:
-        logger.debug("Common Prefix: '%s'", common_whitespace_prefix)
-
-    # Then we trim that much whitespace off all the leading strings.
-    # We tolerate the common prefix not *actually* being common and remove as
-    # much of it as is there, to support tolerate_dedents.
-
-    def first_mismatch(prefix: str, value: str) -> int:
-        """
-        Get the index of the first character in value that does not match the corresponding character in prefix, or the length of the shorter string.
-        """
-        for n, (c1, c2) in enumerate(zip(prefix, value)):
-            if c1 != c2:
-                return n
-        return min(len(prefix), len(value))
-
-    # Trim up to the first mismatch vs. the common prefix if the line starts with a string literal.
-    stripped_lines = [
-        (
-            (
-                cast(
-                    list[Union[str, WDL.Expr.Placeholder]],
-                    [line[0][first_mismatch(common_whitespace_prefix, line[0]) :]],
-                )
-                + line[1:]
-            )
-            if len(line) > 0 and isinstance(line[0], str)
-            else line
-        )
-        for line in lines
-    ]
-    if debug:
-        logger.debug("Stripped Lines: %s", stripped_lines)
-
-    # Then we reassemble the parts and make a new expression.
-    # Build lists and turn the lists into strings later
-    new_parts: list[list[str] | WDL.Expr.Placeholder] = []
-    for i, line in enumerate(stripped_lines):
-        if i > 0:
-            # This is a second line, so we need to tack on a newline.
-            if len(new_parts) > 0 and isinstance(new_parts[-1], list):
-                # Tack on to existing string collection
-                new_parts[-1].append("\n")
-            else:
-                # Make a new string collection
-                new_parts.append(["\n"])
-        if len(line) > 0 and isinstance(line[0], str) and i > 0:
-            # Line starts with a string we need to merge with the last string.
-            # We know the previous line now ends with a string collection, so tack it on.
-            assert isinstance(new_parts[-1], list)
-            new_parts[-1].append(line[0])
-            # Make all the strings into string collections in the rest of the line
-            new_parts += [([x] if isinstance(x, str) else x) for x in line[1:]]
-        else:
-            # No string merge necessary
-            # Make all the strings into string collections in the whole line
-            new_parts += [([x] if isinstance(x, str) else x) for x in line]
-
-    if debug:
-        logger.debug("New Parts: %s", new_parts)
-
-    # Now go back to the alternating strings and placeholders that MiniWDL wants
-    new_parts_merged: list[str | WDL.Expr.Placeholder] = [
-        ("".join(x) if isinstance(x, list) else x) for x in new_parts
-    ]
-
-    if debug:
-        logger.debug("New Parts Merged: %s", new_parts_merged)
-
-    modified = WDL.Expr.String(expression.pos, new_parts_merged, expression.command)
-    # Fake the type checking of the modified expression.
-    # TODO: Make MiniWDL expose a real way to do this?
-    modified._type = expression._type
-    return modified
-
-
 async def toil_read_source(
     uri: str, path: list[str], importer: WDL.Tree.Document | None
 ) -> ReadSourceResult:
@@ -513,7 +315,7 @@ async def toil_read_source(
         tried.append(candidate_uri)
         try:
             # TODO: this is probably sync work that would be better as async work here
-            AbstractJobStore.read_from_url(candidate_uri, destination_buffer)
+            URLAccess.read_from_url(candidate_uri, destination_buffer)
         except Exception as e:
             if isinstance(e, SyntaxError) or isinstance(e, NameError):
                 # These are probably actual problems with the code and not
@@ -917,8 +719,8 @@ def set_shared_fs_path(file: WDL.Value.File, path: str) -> WDL.Value.File:
 
 
 def view_shared_fs_paths(
-    bindings: WDL.Env.Bindings[WDL.Value.Base],
-) -> WDL.Env.Bindings[WDL.Value.Base]:
+    bindings: WDLBindings,
+) -> WDLBindings:
     """
     Given WDL bindings, return a copy where all files have their shared filesystem paths as their values.
     """
@@ -1137,33 +939,50 @@ def choose_human_readable_directory(
 
 def evaluate_decls_to_bindings(
     decls: list[WDL.Tree.Decl],
-    all_bindings: WDL.Env.Bindings[WDL.Value.Base],
+    all_bindings: WDLBindings,
     standard_library: ToilWDLStdLibBase,
     include_previous: bool = False,
     drop_missing_files: bool = False,
-) -> WDL.Env.Bindings[WDL.Value.Base]:
+    expressions_are_defaults: bool = False,
+) -> WDLBindings:
     """
     Evaluate decls with a given bindings environment and standard library.
+    
     Creates a new bindings object that only contains the bindings from the given decls.
     Guarantees that each decl in `decls` can access the variables defined by the previous ones.
+    
     :param all_bindings: Environment to use when evaluating decls
     :param decls: Decls to evaluate
     :param standard_library: Standard library
-    :param include_previous: Whether to include the existing environment in the new returned environment. This will be false for outputs where only defined decls should be included
-    :param drop_missing_files: Whether to coerce nonexistent files to null. The coerced elements will be checked that the transformation is valid.
-    Currently should only be enabled in output sections, see https://github.com/openwdl/wdl/issues/673#issuecomment-2248828116
+    :param include_previous: Whether to include the existing environment in the
+        new returned environment. This will be false for outputs where only
+        defined decls should be included
+    :param drop_missing_files: Whether to coerce nonexistent files to null. The
+        coerced elements will be checked that the transformation is valid.
+        Currently should only be enabled in output sections, see
+        https://github.com/openwdl/wdl/issues/673#issuecomment-2248828116.
+    :param expressions_are_defaults: If True, value expressions in decls are
+        treated as default values, and there may be existing values in the
+        incoming environment that take precedence. If False, each decl is taken
+        to be a fresh definition, and expressions are always evaluated and
+        used.
     :return: New bindings object
     """
     # all_bindings contains current bindings + previous all_bindings
     # bindings only contains the decl bindings themselves so that bindings from other sections prior aren't included
-    bindings: WDL.Env.Bindings[WDL.Value.Base] = WDL.Env.Bindings()
+    bindings: WDLBindings = WDL.Env.Bindings()
     drop_if_missing_with_workdir = partial(
         drop_if_missing, standard_library=standard_library
     )
     for each_decl in decls:
-        output_value = evaluate_defaultable_decl(
-            each_decl, all_bindings, standard_library
-        )
+        if expressions_are_defaults:
+            output_value = evaluate_defaultable_decl(
+                each_decl, all_bindings, standard_library
+            )
+        else:
+            output_value = evaluate_decl(
+                each_decl, all_bindings, standard_library
+            )
         if drop_missing_files:
             dropped_output_value = map_over_typed_files_in_value(
                 output_value, drop_if_missing_with_workdir
@@ -1222,7 +1041,7 @@ class NonDownloadingSize(WDL.StdLib._Size):
                 else:
                     # This is some other kind of remote file.
                     # We need to get its size from the URI.
-                    item_size = AbstractJobStore.get_size(uri)
+                    item_size = URLAccess.get_size(uri)
                     if item_size is None:
                         # User asked for the size and we can't figure it out efficiently, so bail out.
                         raise RuntimeError(f"Attempt to check the size of {uri} failed")
@@ -1245,7 +1064,10 @@ class NonDownloadingSize(WDL.StdLib._Size):
         return WDL.Value.Float(total_size)
 
 
-def extract_workflow_inputs(environment: WDLBindings) -> list[str]:
+def extract_file_values(environment: WDLBindings) -> list[str]:
+    """
+    Get a list of all File object values in the given bindings.
+    """
     filenames = list()
 
     def add_filename(file: WDL.Value.File) -> WDL.Value.File:
@@ -1255,6 +1077,22 @@ def extract_workflow_inputs(environment: WDLBindings) -> list[str]:
     map_over_files_in_bindings(environment, add_filename)
     return filenames
 
+def extract_file_virtualized_values(environment: WDLBindings) -> list[str]:
+    """
+    Get a list of all File object virtualized values in the given bindings.
+
+    If a file hasn't been virtualized, it won't contribute to the list.
+    """
+    values = list()
+
+    def add_value(file: WDL.Value.File) -> WDL.Value.File:
+        value = get_file_virtualized_value(file)
+        if value is not None:
+            values.append(value)
+        return file
+
+    map_over_files_in_bindings(environment, add_value)
+    return values
 
 def convert_files(
     environment: WDLBindings,
@@ -1263,19 +1101,21 @@ def convert_files(
     task_path: str,
 ) -> WDLBindings:
     """
-    Resolve relative-URI files in the given environment convert the file values to a new value made from a given mapping.
+    Fill in the virtualized_value fields for File objects in a WDL environment.
 
-    Will return bindings with file values set to their corresponding relative-URI.
-
-    :param environment: Bindings to evaluate on
-    :return: new bindings object
+    :param environment: Bindings to evaluate on. Will not be modified.
+    :param file_to_id: Maps from imported URI to Toil FileID with the data.
+    :param file_to_data: Maps from WDL-level file calue to metadata about the
+        file, including URI that would have been imported.
+    :return: new bindings object with the annotated File objects in it.
     """
     dir_ids = {t[1] for t in file_to_data.values()}
     dir_to_id = {k: uuid.uuid4() for k in dir_ids}
 
     def convert_file_to_uri(file: WDL.Value.File) -> WDL.Value.File:
         """
-        Calls import_filename to detect if a potential URI exists and imports it. Will modify the File object value to the new URI and tack on the virtualized file.
+        Produce a WDL File with the virtualized_value set to the Toil URI for
+        the already-imported data, but the same value.
         """
         candidate_uri = file_to_data[file.value][0]
         file_id = file_to_id[candidate_uri]
@@ -1352,7 +1192,7 @@ def convert_remote_files(
             tried.append(candidate_uri)
             try:
                 # Try polling existence first.
-                polled_existence = file_source.url_exists(candidate_uri)
+                polled_existence = URLAccess.url_exists(candidate_uri)
                 if polled_existence is False:
                     # Known not to exist
                     logger.debug("URL does not exist: %s", candidate_uri)
@@ -1638,32 +1478,35 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             logger.debug("File has no virtualized value so not changing value")
         return file
 
+    def _resolve_devirtualized_to_uri(self, devirtualized: str) -> str:
+        """
+        Get a URI pointing to whatever URI or divirtualized file path is provided.
+
+        Handles resolving symlinks using in-container paths if necessary.
+        """
+        
+        return Toil.normalize_uri(devirtualized, dir_path=self.execution_dir)
+        
     def _virtualize_file(
         self, file: WDL.Value.File, enforce_existence: bool = True
     ) -> WDL.Value.File:
-        logger.debug("Virtualizing %s", file)
-        # If enforce_existence is true, then if a file is detected as nonexistent, raise an error. Else, let it pass through
         if get_file_virtualized_value(file) is not None:
-            logger.debug("File is marked nonexistent so passing it through")
+            # Already virtualized
             return file
 
-        if enforce_existence is False:
-            # We only want to error on a nonexistent file in the output section
-            # Since we need to virtualize on task boundaries, don't enforce existence if on a boundary
-            if is_standard_url(file.value):
-                file_uri = Toil.normalize_uri(file.value)
-            else:
-                abs_filepath = (
-                    os.path.join(self.execution_dir, file.value)
-                    if self.execution_dir is not None
-                    else os.path.abspath(file.value)
-                )
-                file_uri = Toil.normalize_uri(abs_filepath)
+        logger.debug("Virtualizing %s", file)
 
-            if not AbstractJobStore.url_exists(file_uri):
+        try:
+            # Let the actual virtualization implementation signal a missing file
+            virtualized_filename = self._virtualize_filename(file.value)
+        except FileNotFoundError:
+            if enforce_existence:
+                raise
+            else:
                 logger.debug("File appears nonexistent so marking it nonexistent")
+                # Mark the file nonexistent.
                 return set_file_nonexistent(file, True)
-        virtualized_filename = self._virtualize_filename(file.value)
+
         logger.debug(
             "For file %s got virtualized filename %s", file, virtualized_filename
         )
@@ -1747,7 +1590,7 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             # Open it exclusively
             with open(dest_path, "xb") as dest_file:
                 # And save to it
-                size, executable = AbstractJobStore.read_from_url(filename, dest_file)
+                size, executable = URLAccess.read_from_url(filename, dest_file)
                 if executable:
                     # Set the execute bit in the file's permissions
                     os.chmod(dest_path, os.stat(dest_path).st_mode | stat.S_IXUSR)
@@ -1846,9 +1689,12 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
     @memoize
     def _virtualize_filename(self, filename: str) -> str:
         """
-        from a local path in write_dir, 'virtualize' into the filename as it should present in a File value
+        from a local path or other URL, 'virtualize' into the filename as it should present in a File value.
+
+        New in Toil: the path or URL may not actually exist.
 
         :param filename: Can be a local file path, URL (http, https, s3, gs), or toilfile
+        :raises FileNotFoundError: if the file doesn't actually exist (new addition in Toil over MiniWDL)
         """
 
         if is_toil_url(filename):
@@ -1868,7 +1714,9 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             try:
                 imported = self._file_store.import_file(filename)
             except FileNotFoundError:
-                logger.error(
+                # This might happen because we're also along the code path for
+                # optional file outputs.
+                logger.info(
                     "File at URL %s does not exist or is inaccessible." % filename
                 )
                 raise
@@ -1879,9 +1727,13 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
                     filename,
                     e.code,
                 )
+                # We don't need to handle translating error codes for not
+                # found; import_file does it already. 
                 raise
             if imported is None:
-                # Satisfy mypy, this should never happen though as we don't pass a shared file name (which is the only way import_file returns None)
+                # Satisfy mypy. This should never happen though as we don't
+                # pass a shared file name (which is the only way import_file
+                # returns None)
                 raise RuntimeError("Failed to import URL %s into jobstore." % filename)
             file_basename = os.path.basename(urlsplit(filename).path)
             # Get the URL to the parent directory and use that.
@@ -1890,23 +1742,19 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             dir_id = self._parent_dir_to_ids.setdefault(parent_dir, uuid.uuid4())
             result = pack_toil_uri(imported, self.task_path, dir_id, file_basename)
             logger.debug("Virtualized %s as WDL file %s", filename, result)
-            # We can't put the Toil URI in the virtualized_to_devirtualized cache because it would point to the URL instead of a
-            # local file on the machine, so only store the forward mapping
+            # We can't put the Toil URI in the virtualized_to_devirtualized
+            # cache because it would point to the URL instead of a local file
+            # on the machine, so only store the forward mapping
             self._devirtualized_to_virtualized[filename] = result
             return result
         else:
-            # Otherwise this is a local file and we want to fake it as a Toil file store file
-            # Make it an absolute path
-            parsed = urlparse(filename)
-            if parsed.scheme == "file":
-                # conversion was already done by normalize_uri
-                abs_filename = unquote(parsed.path)
-            elif self.execution_dir is not None:
-                # To support relative paths from execution directory, join the execution dir and filename
-                # If filename is already an abs path, join() will not do anything
-                abs_filename = os.path.join(self.execution_dir, filename)
-            else:
-                abs_filename = os.path.abspath(filename)
+            # Otherwise this is a local file name or URI and we want to fake it
+            # as a Toil file store file
+
+            # Convert to a properly-absolutized file URI
+            file_uri = Toil.normalize_uri(filename, dir_path=self.execution_dir)
+            # Extract the absolute path name
+            abs_filename = unquote(urlsplit(file_uri).path)
 
             if abs_filename in self._devirtualized_to_virtualized:
                 # This is a previously devirtualized thing so we can just use the
@@ -1917,6 +1765,9 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
                 )
                 return result
 
+            if not os.path.exists(abs_filename):
+                raise FileNotFoundError(abs_filename)
+
             file_id = self._file_store.writeGlobalFile(abs_filename)
 
             file_dir = os.path.dirname(abs_filename)
@@ -1946,6 +1797,51 @@ class ToilWDLStdLibWorkflow(ToilWDLStdLibBase):
 
         self._miniwdl_cache: Optional[WDL.runtime.cache.CallCache] = None
 
+    def _virtualize_file(
+        self, file: WDL.Value.File, enforce_existence: bool = True
+    ) -> WDL.Value.File:
+        # When a workflow coerces a string path or file: URI to a File at
+        # workflow scope, we need to fill in the cache filesystem path.
+        if (
+            get_file_virtualized_value(file) is None
+            and get_shared_fs_path(file) is None
+            and (
+                not is_any_url(file.value)
+                or is_file_url(file.value)
+            )
+        ):
+            # This is a never-virtualized file that is a file path or URI and
+            # has no shared FS path associated with it. We just made it at
+            # workflow scope. (If it came from a task, it would have a
+            # virtualized value already.)
+
+            # If we are loading it at workflow scope, the file path can be used
+            # as the cache path.
+
+            if not is_any_url(file.value):
+                # Handle file path
+                cache_path = file.value
+            else:
+                # Handle pulling path out of file URI
+                cache_path = unquote(urlsplit(file.value).path)
+
+            # Apply the path
+            file = set_shared_fs_path(file, cache_path)
+
+            logger.info(
+                "Applied shared filesystem path %s to File %s that appears to "
+                "have been coerced from String at workflow scope.",
+                cache_path,
+                file
+            )
+
+        # Do the virtualization
+        return super()._virtualize_file(file, enforce_existence)
+
+        # TODO: If the workflow coerces a File to a String and back again, we
+        # should have some way to recover the toilfile: URL it had in the job
+        # store to avoid re-importing it.
+
     # This needs to be hash-compatible with MiniWDL.
     # MiniWDL hooks _virtualize_filename
     # <https://github.com/chanzuckerberg/miniwdl/blob/475dd3f3784d1390e6a0e880d43316a620114de3/WDL/runtime/workflow.py#L699-L729>,
@@ -1999,7 +1895,7 @@ class ToilWDLStdLibWorkflow(ToilWDLStdLibBase):
             )
             # Make an environment of "file_sha256" to that as a WDL string, and
             # digest that, and make a write_ cache key. No need to transform to
-            # shared FS paths sonce no paths are in it.
+            # shared FS paths since no paths are in it.
             log_bindings(
                 logger.debug, "Digesting file bindings:", [file_input_bindings]
             )
@@ -2346,6 +2242,8 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
         filenames.
         """
 
+        logger.debug("WDL task outputs stdlib asked to virtualize %s", filename)
+
         if not is_any_url(filename) and not filename.startswith("/"):
             # We are getting a bare relative path on the supposedly devirtualized side.
             # Find a real path to it relative to the current directory override.
@@ -2394,8 +2292,12 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
                 logger.error(
                     "Handling broken symlink %s ultimately to %s", filename, here
                 )
+                # This should produce a FileNotFoundError since we think of
+                # broken symlinks as nonexistent.
+                raise FileNotFoundError(filename)
             filename = here
-
+        
+        logger.debug("WDL task outputs stdlib thinks we really need to virtualize %s", filename)
         return super()._virtualize_filename(filename)
 
 
@@ -2450,11 +2352,15 @@ def evaluate_decl(
     """
     Evaluate the expression of a declaration node, or raise an error.
     """
-
-    return evaluate_named_expression(
-        node, node.name, node.type, node.expr, environment, stdlib
-    )
-
+    try:
+        return evaluate_named_expression(
+            node, node.name, node.type, node.expr, environment, stdlib
+        )
+    except Exception:
+        # If something goes wrong, dump.
+        logger.exception("Evaluation failed for %s", node)
+        log_bindings(logger.error, "Statement was evaluated in:", [environment])
+        raise
 
 def evaluate_call_inputs(
     context: WDL.Error.SourceNode | WDL.Error.SourcePosition,
@@ -2497,33 +2403,28 @@ def evaluate_defaultable_decl(
     If the name of the declaration is already defined in the environment, return its value. Otherwise, return the evaluated expression.
     """
 
-    try:
-        if (
-            node.name in environment
-            and not isinstance(environment[node.name], WDL.Value.Null)
-        ) or (
-            isinstance(environment.get(node.name), WDL.Value.Null)
-            and node.type.optional
-        ):
-            logger.debug("Name %s is already defined, not using default", node.name)
-            if not isinstance(environment[node.name].type, type(node.type)):
-                return environment[node.name].coerce(node.type)
-            else:
-                return environment[node.name]
+    if (
+        node.name in environment
+        and not isinstance(environment[node.name], WDL.Value.Null)
+    ) or (
+        isinstance(environment.get(node.name), WDL.Value.Null)
+        and node.type.optional
+    ):
+        logger.debug("Name %s is already defined, not using default", node.name)
+        if not isinstance(environment[node.name].type, type(node.type)):
+            return environment[node.name].coerce(node.type)
         else:
-            if node.type is not None and not node.type.optional and node.expr is None:
-                # We need a value for this but there isn't one.
-                raise WDL.Error.EvalError(
-                    node,
-                    f"Value for {node.name} was not provided and no default value is available",
-                )
-            logger.info("Defaulting %s to %s", node.name, node.expr)
-            return evaluate_decl(node, environment, stdlib)
-    except Exception:
-        # If something goes wrong, dump.
-        logger.exception("Evaluation failed for %s", node)
-        log_bindings(logger.error, "Statement was evaluated in:", [environment])
-        raise
+            return environment[node.name]
+    else:
+        if node.type is not None and not node.type.optional and node.expr is None:
+            # We need a value for this but there isn't one.
+            raise WDL.Error.EvalError(
+                node,
+                f"Value for {node.name} was not provided and no default value is available",
+            )
+        logger.info("Defaulting %s to %s", node.name, node.expr)
+        return evaluate_decl(node, environment, stdlib)
+    
 
 
 # TODO: make these stdlib methods???
@@ -2535,7 +2436,7 @@ def devirtualize_files(
     that are actually available to command line commands.
     The same virtual file always maps to the same devirtualized filename even with duplicates
     """
-    logger.info("Devirtualizing files")
+    logger.debug("Devirtualizing files")
     return map_over_files_in_bindings(environment, stdlib._devirtualize_file)
 
 
@@ -2546,12 +2447,35 @@ def virtualize_files(
     Make sure all the File values embedded in the given bindings point to files
     that are usable from other machines.
     """
-    logger.info("Virtualizing files")
+    logger.debug("Virtualizing files")
     virtualize_func = partial(
         stdlib._virtualize_file, enforce_existence=enforce_existence
     )
     return map_over_files_in_bindings(environment, virtualize_func)
 
+def delete_dead_files(internal_bindings: WDLBindings, live_bindings_list: list[WDLBindings], file_store: AbstractFileStore) -> None:
+    """
+    Delete any files that in the given bindings but not in the live list.
+
+    Operates on the virtualized values of File objects anywhere in the bindings.
+    """
+
+    # Get all the files in the first bindings and not any of the others.
+    unused_files = set(
+        extract_file_virtualized_values(internal_bindings)
+    ).difference(
+        *(
+            extract_file_virtualized_values(bindings)
+            for bindings in live_bindings_list
+        )
+    )
+
+    for file_uri in unused_files:
+        # Delete them
+        if is_toil_url(file_uri):
+            logger.debug("Delete file %s that is not needed", file_uri)
+            file_id, _, _, _ = unpack_toil_uri(file_uri)
+            file_store.deleteGlobalFile(file_id)
 
 def add_paths(task_container: TaskContainer, host_paths: Iterable[str]) -> None:
     """
@@ -2612,7 +2536,7 @@ def drop_if_missing(
 
     if filename is not None and is_any_url(filename):
         try:
-            if filename.startswith(TOIL_URI_SCHEME) or AbstractJobStore.url_exists(
+            if filename.startswith(TOIL_URI_SCHEME) or URLAccess.url_exists(
                 filename
             ):
                 # We assume anything in the filestore actually exists.
@@ -2728,64 +2652,52 @@ def map_over_files_in_binding(
         binding.info,
     )
 
+def remove_expr_from_value(value: WDL.Value.Base) -> WDL.Value.Base:
+    """
+    Remove the expression from a WDL value
+    :param value: Original WDL value
+    :return: New WDL value without the expr field
+    """
+    # TODO: This is an extra copy that we could get rid of by dropping the immutability idea
+    def predicate(value: WDL.Value.Base) -> WDL.Value.Base:
+        # Do a shallow copy to preserve immutability
+        new_value = copy.copy(value)
+        if value.expr:
+            # We use a Null expr instead of None here, because when evaluating an expression, 
+            # MiniWDL applies that expression to the result value *and* all values it contains that
+            # have None expressions. Using a Null expression here protects nested values that 
+            # didn't really get created by the current expression from being attributed to it, while
+            # still cutting the reference to the parsed WDL document.
+            new_value._expr = WDL.Expr.Null(value.expr.pos)
+        else:
+            new_value._expr = value.expr
+        return new_value
+    return map_over_typed_value(value, predicate)
 
-# TODO: We want to type this to say, for anything descended from a WDL type, we
-# return something descended from the same WDL type or a null. But I can't
-# quite do that with generics, since you could pass in some extended WDL value
-# type we've never heard of and expect to get one of those out.
-#
-# For now we assume that any types extending the WDL value types will implement
-# compatible constructors.
-def map_over_typed_files_in_value(
-    value: WDL.Value.Base, transform: Callable[[WDL.Value.File], WDL.Value.File | None]
-) -> WDL.Value.Base:
-    """
-    Run all File values embedded in the given value through the given
-    transformation function.
-
-    The transformation function must not mutate the original File.
-
-    If the transform returns None, the file value is changed to Null.
-
-    The transform has access to the type information for the value, so it knows
-    if it may return None, depending on if the value is optional or not.
 
-    The transform is *allowed* to return None only if the mapping result won't
-    actually be used, to allow for scans. So error checking needs to be part of
-    the transform itself.
+def map_over_typed_value(value: WDL.Value.Base, transform: Callable[[WDL.Value.Base], WDL.Value.Base]) -> WDL.Value.Base:
     """
-    if isinstance(value, WDL.Value.File):
-        # This is a file so we need to process it
-        orig_file_value = value.value
-        new_file = transform(value)
-        assert (
-            value.value == orig_file_value
-        ), "Transformation mutated the original File"
-        if new_file is None:
-            # Assume the transform checked types if we actually care about the
-            # result.
-            logger.warning("File %s became Null", value)
-            return WDL.Value.Null()
-        else:
-            # Make whatever the value is around the new path.
-            # TODO: why does this need casting?
-            return new_file
-    elif isinstance(value, WDL.Value.Array):
+    Apply a transform to a WDL value and all contained WDL values.
+    :param value: WDL value to transform
+    :param transform: Function that takes a WDL value and returns a new WDL value
+    :return: New transformed WDL value
+    """
+    if isinstance(value, WDL.Value.Array):
         # This is an array, so recurse on the items
-        return WDL.Value.Array(
+        value = WDL.Value.Array(
             value.type.item_type,
-            [map_over_typed_files_in_value(v, transform) for v in value.value],
+            [map_over_typed_value(v, transform) for v in value.value],
             value.expr,
         )
     elif isinstance(value, WDL.Value.Map):
         # This is a map, so recurse on the members of the items, which are tuples (but not wrapped as WDL Pair objects)
         # TODO: Can we avoid a cast in a comprehension if we get MyPy to know that each pair is always a 2-element tuple?
-        return WDL.Value.Map(
+        value = WDL.Value.Map(
             value.type.item_type,
             [
                 cast(
                     tuple[WDL.Value.Base, WDL.Value.Base],
-                    tuple(map_over_typed_files_in_value(v, transform) for v in pair),
+                    tuple(map_over_typed_value(v, transform) for v in pair),
                 )
                 for pair in value.value
             ],
@@ -2793,29 +2705,74 @@ def map_over_typed_files_in_value(
         )
     elif isinstance(value, WDL.Value.Pair):
         # This is a pair, so recurse on the left and right items
-        return WDL.Value.Pair(
+        value = WDL.Value.Pair(
             value.type.left_type,
             value.type.right_type,
             cast(
                 tuple[WDL.Value.Base, WDL.Value.Base],
-                tuple(map_over_typed_files_in_value(v, transform) for v in value.value),
+                tuple(map_over_typed_value(v, transform) for v in value.value),
             ),
             value.expr,
         )
     elif isinstance(value, WDL.Value.Struct):
         # This is a struct, so recurse on the values in the backing dict
-        return WDL.Value.Struct(
+        value = WDL.Value.Struct(
             cast(Union[WDL.Type.StructInstance, WDL.Type.Object], value.type),
             {
-                k: map_over_typed_files_in_value(v, transform)
+                k: map_over_typed_value(v, transform)
                 for k, v in value.value.items()
             },
             value.expr,
         )
-    else:
-        # All other kinds of value can be passed through unmodified.
+    # Run the predicate on the final value
+    return transform(value)
+
+
+# TODO: We want to type this to say, for anything descended from a WDL type, we
+# return something descended from the same WDL type or a null. But I can't
+# quite do that with generics, since you could pass in some extended WDL value
+# type we've never heard of and expect to get one of those out.
+#
+# For now we assume that any types extending the WDL value types will implement
+# compatible constructors.
+def map_over_typed_files_in_value(
+    value: WDL.Value.Base, transform: Callable[[WDL.Value.File], WDL.Value.File | None]
+) -> WDL.Value.Base:
+    """
+    Run all File values embedded in the given value through the given
+    transformation function.
+
+    The transformation function must not mutate the original File.
+
+    If the transform returns None, the file value is changed to Null.
+
+    The transform has access to the type information for the value, so it knows
+    if it may return None, depending on if the value is optional or not.
+
+    The transform is *allowed* to return None only if the mapping result won't
+    actually be used, to allow for scans. So error checking needs to be part of
+    the transform itself.
+    """
+    def predicate(value: WDL.Value.Base) -> WDL.Value.Base:
+        if isinstance(value, WDL.Value.File):
+            # This is a file so we need to process it
+            orig_file_value = value.value
+            new_file = transform(value)
+            assert (
+                value.value == orig_file_value
+            ), "Transformation mutated the original File"
+            if new_file is None:
+                # Assume the transform checked types if we actually care about the
+                # result.
+                logger.warning("File %s became Null", value)
+                return WDL.Value.Null()
+            else:
+                # Make whatever the value is around the new path.
+                return new_file
         return value
 
+    return map_over_typed_value(value, predicate)
+
 
 def ensure_null_files_are_nullable(
     value: WDL.Value.Base, original_value: WDL.Value.Base, expected_type: WDL.Type.Base
@@ -2958,6 +2915,11 @@ class WDLBaseJob(Job):
         logger.debug("Overlay %s after %s", overlay, self)
         self._postprocessing_steps.append(("overlay", overlay))
 
+    def remove_expr_from_bindings(self, bindings: WDLBindings) -> WDLBindings:
+        # We have to throw out the expressions because they drag the entire WDL document into the WDL outputs
+        # which causes duplicate pickling and linear growth in scatter memory usage
+        return bindings.map(lambda b: WDL.Env.Binding(b.name, remove_expr_from_value(b.value), b.info))
+
     def postprocess(self, bindings: WDLBindings) -> WDLBindings:
         """
         Apply queued changes to bindings.
@@ -2994,7 +2956,7 @@ class WDLBaseJob(Job):
                 bindings = combine_bindings([bindings.subtract(argument), argument])
             else:
                 raise RuntimeError(f"Unknown postprocessing action {action}")
-
+        bindings = self.remove_expr_from_bindings(bindings)
         return bindings
 
     def defer_postprocessing(self, other: WDLBaseJob) -> None:
@@ -3025,6 +2987,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
         self,
         task: WDL.Tree.Task,
         prev_node_results: Sequence[Promised[WDLBindings]],
+        enclosing_bindings: WDLBindings,
         task_id: list[str],
         wdl_options: WDLContext,
         **kwargs: Any,
@@ -3032,6 +2995,11 @@ class WDLTaskWrapperJob(WDLBaseJob):
         """
         Make a new job to determine resources and run a task.
 
+        :param enclosing_bindings: Bindings in the enclosing section,
+            containing files not to clean up. Files that are passed as inputs
+            but not uses as outputs or present in the enclosing section
+            bindings will be deleted after the task call completes.
+
         :param namespace: The namespace that the task's *contents* exist in.
                The caller has alredy added the task's own name.
         """
@@ -3052,6 +3020,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
 
         self._task = task
         self._prev_node_results = prev_node_results
+        self._enclosing_bindings = enclosing_bindings
         self._task_id = task_id
 
     @report_wdl_errors("evaluate task code", exit=True)
@@ -3091,17 +3060,34 @@ class WDLTaskWrapperJob(WDLBaseJob):
             # TODO: What if the same file is passed through several tasks, and
             # we get cache hits on those tasks? Won't we upload it several
             # times?
+
+            # Load output bindings from the cache
+            cached_bindings = virtualize_files(
+                cached_result, standard_library, enforce_existence=False
+            )
+
+            # Throw away anything input but not available outside the call or
+            # output.
+            delete_dead_files(
+                bindings,
+                [cached_bindings, self._enclosing_bindings],
+                file_store
+            )
+
+            # Postprocess and ship the output bindings.
             return self.postprocess(
-                virtualize_files(
-                    cached_result, standard_library, enforce_existence=False
-                )
+                cached_bindings
             )
 
         if self._task.inputs:
             logger.debug("Evaluating task code")
             # Evaluate all the inputs that aren't pre-set
             bindings = evaluate_decls_to_bindings(
-                self._task.inputs, bindings, standard_library, include_previous=True
+                self._task.inputs,
+                bindings,
+                standard_library,
+                include_previous=True,
+                expressions_are_defaults=True
             )
         if self._task.postinputs:
             # Evaluate all the postinput decls.
@@ -3231,6 +3217,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
             virtualize_files(
                 runtime_bindings, standard_library, enforce_existence=False
             ),
+            self._enclosing_bindings,
             self._task_id,
             cores=runtime_cores or self.cores,
             memory=runtime_memory or self.memory,
@@ -3266,6 +3253,7 @@ class WDLTaskJob(WDLBaseJob):
         task: WDL.Tree.Task,
         task_internal_bindings: Promised[WDLBindings],
         runtime_bindings: Promised[WDLBindings],
+        enclosing_bindings: WDLBindings,
         task_id: list[str],
         mount_spec: dict[str | None, int],
         wdl_options: WDLContext,
@@ -3275,6 +3263,9 @@ class WDLTaskJob(WDLBaseJob):
         """
         Make a new job to run a task.
 
+        :param enclosing_bindings: Bindings outside the workflow call, with
+            files that should not be cleaned up at the end of the task.
+
         :param namespace: The namespace that the task's *contents* exist in.
                The caller has alredy added the task's own name.
         """
@@ -3298,6 +3289,7 @@ class WDLTaskJob(WDLBaseJob):
         self._task = task
         self._task_internal_bindings = task_internal_bindings
         self._runtime_bindings = runtime_bindings
+        self._enclosing_bindings = enclosing_bindings
         self._task_id = task_id
         self._cache_key = cache_key
         self._mount_spec = mount_spec
@@ -3646,6 +3638,8 @@ class WDLTaskJob(WDLBaseJob):
                     "is not yet implemented in the MiniWDL Docker "
                     "containerization implementation."
                 )
+            if runtime_bindings.has_binding("memory") and human2bytes(runtime_bindings.resolve("memory").value) < human2bytes("4MiB"):
+                    runtime_bindings.resolve("memory").value = "4MiB"
         else:
             raise RuntimeError(
                 f"Could not find a working container engine to use; told to use {self._wdl_options.get('container')}"
@@ -3878,7 +3872,7 @@ class WDLTaskJob(WDLBaseJob):
                     self._task,
                     "command",
                     WDL.Type.String(),
-                    remove_common_leading_whitespace(self._task.command),
+                    self._task.command,
                     contained_bindings,
                     command_library,
                 )
@@ -4056,6 +4050,18 @@ class WDLTaskJob(WDLBaseJob):
                 miniwdl_config=miniwdl_config,
             )
 
+        # Clean up anything from the task call input: block or the runtime
+        # section that isn't getting output or available in the enclosing
+        # section. Runtime sections aren't meant to have files, but nothing
+        # actually stops them from being there.
+        delete_dead_files(
+            combine_bindings([bindings, runtime_bindings]),
+            [output_bindings, self._enclosing_bindings],
+            file_store
+        )
+        # If File objects somehow made it to the runtime block they shouldn't
+        # have been virtualized so don't bother with them.
+
         # Do postprocessing steps to e.g. apply namespaces.
         output_bindings = self.postprocess(output_bindings)
 
@@ -4108,7 +4114,8 @@ class WDLWorkflowNodeJob(WDLBaseJob):
             logger.info("Setting %s to %s", self._node.name, self._node.expr)
             value = evaluate_decl(self._node, incoming_bindings, standard_library)
             bindings = incoming_bindings.bind(self._node.name, value)
-            return self.postprocess(bindings)
+            # TODO: Only virtualize the new binding
+            return self.postprocess(virtualize_files(bindings, standard_library, enforce_existence=False))
         elif isinstance(self._node, WDL.Tree.Call):
             # This is a call of a task or workflow
 
@@ -4129,6 +4136,8 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 standard_library,
                 inputs_mapping,
             )
+            # Prepare call inputs to move to another node
+            input_bindings = virtualize_files(input_bindings, standard_library, enforce_existence=False)
 
             # Bindings may also be added in from the enclosing workflow inputs
             # TODO: this is letting us also inject them from the workflow body.
@@ -4146,6 +4155,7 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 subjob: WDLBaseJob = WDLWorkflowJob(
                     self._node.callee,
                     [input_bindings, passed_down_bindings],
+                    incoming_bindings,
                     self._node.callee_id,
                     wdl_options=wdl_options,
                     local=True,
@@ -4156,6 +4166,7 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 subjob = WDLTaskWrapperJob(
                     self._node.callee,
                     [input_bindings, passed_down_bindings],
+                    incoming_bindings,
                     self._node.callee_id,
                     wdl_options=wdl_options,
                     local=True,
@@ -4257,7 +4268,8 @@ class WDLWorkflowNodeListJob(WDLBaseJob):
                     node, "Unimplemented WorkflowNode: " + str(type(node))
                 )
 
-        return self.postprocess(current_bindings)
+        # TODO: Only virtualize the new bindings created
+        return self.postprocess(virtualize_files(current_bindings, standard_library, enforce_existence=False))
 
 
 class WDLCombineBindingsJob(WDLBaseJob):
@@ -4792,6 +4804,12 @@ class WDLScatterJob(WDLSectionJob):
                 [(p, p) for p in standard_library.get_local_paths()]
             )
 
+        # Set the exprs of the WDL values to WDL.Expr.Null to reduce the memory footprint. This got set from evaluate_named_expression
+        # because any evaluation on an expression will mutate child values of the result values of the expression, and we had not
+        # processed it yet by this point as the bindings from input environment and WDLWorkflowJob do not get processing and postprocessing
+        # ran respectively
+        bindings = self.remove_expr_from_bindings(bindings)
+
         if not isinstance(scatter_value, WDL.Value.Array):
             raise RuntimeError(
                 "The returned value from a scatter is not an Array type."
@@ -4804,6 +4822,8 @@ class WDLScatterJob(WDLSectionJob):
             # duration of the body.
             local_bindings: WDLBindings = WDL.Env.Bindings()
             local_bindings = local_bindings.bind(self._scatter.variable, item)
+            # Remove expr from new scatter binding
+            local_bindings = self.remove_expr_from_bindings(local_bindings)
             # TODO: We need to turn values() into a list because MyPy seems to
             # think a dict_values isn't a Sequence. This is a waste of time to
             # appease MyPy but probably better than a cast?
@@ -5020,6 +5040,7 @@ class WDLWorkflowJob(WDLSectionJob):
         self,
         workflow: WDL.Tree.Workflow,
         prev_node_results: Sequence[Promised[WDLBindings]],
+        enclosing_bindings: WDLBindings,
         workflow_id: list[str],
         wdl_options: WDLContext,
         **kwargs: Any,
@@ -5028,6 +5049,13 @@ class WDLWorkflowJob(WDLSectionJob):
         Create a subtree that will run a WDL workflow. The job returns the
         return value of the workflow.
 
+        :param prev_node_results: Bindings fed into the workflow call as inputs.
+
+        :param enclosing_bindings: Bindings in the enclosing section,
+            containing files not to clean up. Files that are passed as inputs
+            but not uses as outputs or present in the enclosing section
+            bindings will be deleted after the workflow call completes.
+
         :param namespace: the namespace that the workflow's *contents* will be
                in. Caller has already added the workflow's own name.
         """
@@ -5044,6 +5072,7 @@ class WDLWorkflowJob(WDLSectionJob):
 
         self._workflow = workflow
         self._prev_node_results = prev_node_results
+        self._enclosing_bindings = enclosing_bindings
         self._workflow_id = workflow_id
 
     @report_wdl_errors("run workflow")
@@ -5084,6 +5113,7 @@ class WDLWorkflowJob(WDLSectionJob):
                     bindings,
                     standard_library,
                     include_previous=True,
+                    expressions_are_defaults=True,
                 )
             finally:
                 # Report all files are downloaded now that all expressions are evaluated.
@@ -5095,11 +5125,13 @@ class WDLWorkflowJob(WDLSectionJob):
         # Make jobs to run all the parts of the workflow
         sink = self.create_subgraph(self._workflow.body, [], bindings)
 
-        # To support the all call outputs feature, run an outputs job even if
-        # we have a declared but empty outputs section.
+        # To support the all call outputs feature and cleanup of files created
+        # in input: blocks, run an outputs job even if we have a declared but
+        # empty outputs section.
         outputs_job = WDLOutputsJob(
             self._workflow,
             sink.rv(),
+            self._enclosing_bindings,
             wdl_options=self._wdl_options,
             cache_key=cache_key,
             local=True,
@@ -5121,6 +5153,7 @@ class WDLOutputsJob(WDLBaseJob):
         self,
         workflow: WDL.Tree.Workflow,
         bindings: Promised[WDLBindings],
+        enclosing_bindings: WDLBindings,
         wdl_options: WDLContext,
         cache_key: str | None = None,
         **kwargs: Any,
@@ -5128,6 +5161,11 @@ class WDLOutputsJob(WDLBaseJob):
         """
         Make a new WDLWorkflowOutputsJob for the given workflow, with the given set of bindings after its body runs.
 
+        :param bindings: Bindings after execution of the workflow body.
+
+        :param enclosing_bindings: Bindings outside the workflow call, with
+            files that should not be cleaned up at the end of the workflow.
+
         :param cache_key: If set and storing into the call cache is on, will
                cache the workflow execution result under the given key in a
                MiniWDL-compatible way.
@@ -5135,6 +5173,7 @@ class WDLOutputsJob(WDLBaseJob):
         super().__init__(wdl_options=wdl_options, **kwargs)
 
         self._bindings = bindings
+        self._enclosing_bindings = enclosing_bindings
         self._workflow = workflow
         self._cache_key = cache_key
 
@@ -5150,9 +5189,8 @@ class WDLOutputsJob(WDLBaseJob):
 
         try:
             if self._workflow.outputs is not None:
-                # Output section is declared and is nonempty, so evaluate normally
-
-                # Combine the bindings from the previous job
+                # Output section is declared and is nonempty, so evaluate normally.
+                # Don't drop nonexistent files here; we do that later.
                 output_bindings = evaluate_decls_to_bindings(
                     self._workflow.outputs, unwrap(self._bindings), standard_library
                 )
@@ -5163,7 +5201,8 @@ class WDLOutputsJob(WDLBaseJob):
             if self._workflow.outputs is None or self._wdl_options.get(
                 "all_call_outputs", False
             ):
-                # The output section is not declared, or we want to keep task outputs anyway.
+                # The output section is not declared, or we want to keep task
+                # outputs anyway on top of an already-evaluated output section.
 
                 # Get all task outputs and return that
                 # First get all task output names
@@ -5194,16 +5233,6 @@ class WDLOutputsJob(WDLBaseJob):
                         output_bindings = output_bindings.bind(
                             binding.name, binding.value
                         )
-            else:
-                # Output section is declared and is nonempty, so evaluate normally
-
-                # Combine the bindings from the previous job
-                output_bindings = evaluate_decls_to_bindings(
-                    self._workflow.outputs,
-                    unwrap(self._bindings),
-                    standard_library,
-                    drop_missing_files=True,
-                )
         finally:
             # We don't actually know when all our files are downloaded since
             # anything we evaluate might devirtualize inside any expression.
@@ -5222,13 +5251,27 @@ class WDLOutputsJob(WDLBaseJob):
             output_bindings, standard_library=standard_library
         )
 
+        # TODO: Unify the rest of this with task output managment somehow
+
+        # Upload any files in the outputs if not uploaded already.
+        # We need this because it's possible to create new files in a workflow
+        # outputs section.
+        output_bindings = virtualize_files(output_bindings, standard_library)
+
         if self._cache_key is not None:
             output_bindings = fill_execution_cache(
                 self._cache_key, output_bindings, file_store, self._wdl_options
             )
 
-        return self.postprocess(output_bindings)
+        # Let Files that are not output or available outside the call go out of
+        # scope.
+        delete_dead_files(
+            unwrap(self._bindings),
+            [output_bindings, self._enclosing_bindings],
+            file_store
+        )
 
+        return self.postprocess(output_bindings)
 
 class WDLStartJob(WDLSectionJob):
     """
@@ -5263,18 +5306,24 @@ class WDLStartJob(WDLSectionJob):
         if isinstance(self._target, WDL.Tree.Workflow):
             # Create a workflow job. We rely in this to handle entering the input
             # namespace if needed, or handling free-floating inputs.
+            # Pass top-level inputs as enclosing section inputs to avoid
+            # bothering to separately delete them.
             job: WDLBaseJob = WDLWorkflowJob(
                 self._target,
                 [inputs],
+                inputs,
                 [self._target.name],
                 wdl_options=self._wdl_options,
                 local=True,
             )
         else:
             # There is no workflow. Create a task job.
+            # Pass top-level inputs as enclosing section inputs to avoid
+            # bothering to separately delete them.
             job = WDLTaskWrapperJob(
                 self._target,
                 [inputs],
+                inputs,
                 [self._target.name],
                 wdl_options=self._wdl_options,
                 local=True,
@@ -5330,7 +5379,7 @@ class WDLImportWrapper(WDLSectionJob):
         wdl_options: WDLContext,
         inputs_search_path: list[str],
         import_remote_files: bool,
-        import_workers_threshold: ParseableIndivisibleResource,
+        import_workers_batchsize: ParseableIndivisibleResource,
         import_workers_disk: ParseableIndivisibleResource,
         **kwargs: Any,
     ):
@@ -5344,11 +5393,11 @@ class WDLImportWrapper(WDLSectionJob):
         self._target = target
         self._inputs_search_path = inputs_search_path
         self._import_remote_files = import_remote_files
-        self._import_workers_threshold = import_workers_threshold
+        self._import_workers_batchsize = import_workers_batchsize
         self._import_workers_disk = import_workers_disk
 
     def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
-        filenames = extract_workflow_inputs(self._inputs)
+        filenames = extract_file_values(self._inputs)
         file_to_data = get_file_sizes(
             filenames,
             file_store.jobStore,
@@ -5356,7 +5405,7 @@ class WDLImportWrapper(WDLSectionJob):
             include_remote_files=self._import_remote_files,
             execution_dir=self._wdl_options.get("execution_dir")
         )
-        imports_job = ImportsJob(file_to_data, self._import_workers_threshold, self._import_workers_disk)
+        imports_job = ImportsJob(file_to_data, self._import_workers_batchsize, self._import_workers_disk)
         self.addChild(imports_job)
         install_imports_job = WDLInstallImportsJob(
             self._target.name, self._inputs, imports_job.rv()
@@ -5388,7 +5437,7 @@ def make_root_job(
             wdl_options=wdl_options,
             inputs_search_path=inputs_search_path,
             import_remote_files=options.reference_inputs,
-            import_workers_threshold=options.import_workers_threshold,
+            import_workers_batchsize=options.import_workers_batchsize,
             import_workers_disk=options.import_workers_disk
         )
     else:
@@ -5445,61 +5494,106 @@ def main() -> None:
         wdl_uri, trs_spec = resolve_workflow(options.wdl_uri, supported_languages={"WDL"})
 
         with Toil(options, workflow_name=trs_spec or wdl_uri, trs_spec=trs_spec) as toil:
-            if options.restart:
-                output_bindings = toil.restart()
+            # TODO: Move all the input parsing outside the Toil context
+            # manager to avoid leaving a job store behind if the workflow
+            # can't start.
+
+            # Both start and restart need us to have the workflow and the
+            # wdl_options WDLContext.
+
+            # MiniWDL load code internally uses asyncio.get_event_loop()
+            # which might not get an event loop if somebody has ever called
+            # set_event_loop. So we need to make sure an event loop is
+            # available.
+            asyncio.set_event_loop(asyncio.new_event_loop())
+
+            # Load the WDL document.
+            document: WDL.Tree.Document = WDL.load(
+                wdl_uri,
+                read_source=toil_read_source,
+                check_quant=options.quant_check
+            )
+
+            # See if we're going to run a workflow or a task
+            target: WDL.Tree.Workflow | WDL.Tree.Task
+            if document.workflow:
+                target = document.workflow
+            elif len(document.tasks) == 1:
+                target = document.tasks[0]
+            elif len(document.tasks) > 1:
+                raise WDL.Error.InputError(
+                    "Multiple tasks found with no workflow! Either add a workflow or keep one task."
+                )
             else:
-                # TODO: Move all the input parsing outside the Toil context
-                # manager to avoid leaving a job store behind if the workflow
-                # can't start.
-                
-                # MiniWDL load code internally uses asyncio.get_event_loop()
-                # which might not get an event loop if somebody has ever called
-                # set_event_loop. So we need to make sure an event loop is
-                # available.
-                asyncio.set_event_loop(asyncio.new_event_loop())
-
-                # Load the WDL document.
-                document: WDL.Tree.Document = WDL.load(
-                    wdl_uri,
-                    read_source=toil_read_source,
+                raise WDL.Error.InputError("WDL document is empty!")
+
+            if "croo_out_def" in target.meta:
+                # This workflow or task wants to have its outputs
+                # "organized" by the Cromwell Output Organizer:
+                # <https://github.com/ENCODE-DCC/croo>.
+                #
+                # TODO: We don't support generating anything that CROO can read.
+                logger.warning(
+                    "This WDL expects to be used with the Cromwell Output Organizer (croo) <https://github.com/ENCODE-DCC/croo>. Toil cannot yet produce the outputs that croo requires. You will not be able to use croo on the output of this Toil run!"
                 )
 
-                # See if we're going to run a workflow or a task
-                target: WDL.Tree.Workflow | WDL.Tree.Task
-                if document.workflow:
-                    target = document.workflow
-                elif len(document.tasks) == 1:
-                    target = document.tasks[0]
-                elif len(document.tasks) > 1:
-                    raise WDL.Error.InputError(
-                        "Multiple tasks found with no workflow! Either add a workflow or keep one task."
-                    )
-                else:
-                    raise WDL.Error.InputError("WDL document is empty!")
-
-                if "croo_out_def" in target.meta:
-                    # This workflow or task wants to have its outputs
-                    # "organized" by the Cromwell Output Organizer:
-                    # <https://github.com/ENCODE-DCC/croo>.
-                    #
-                    # TODO: We don't support generating anything that CROO can read.
+                # But we can assume that we need to preserve individual
+                # taks outputs since the point of CROO is fetching those
+                # from Cromwell's output directories.
+                #
+                # This isn't quite WDL spec compliant but it will rescue
+                # runs of the popular
+                # <https://github.com/ENCODE-DCC/atac-seq-pipeline>
+                if options.all_call_outputs is None:
                     logger.warning(
-                        "This WDL expects to be used with the Cromwell Output Organizer (croo) <https://github.com/ENCODE-DCC/croo>. Toil cannot yet produce the outputs that croo requires. You will not be able to use croo on the output of this Toil run!"
+                        "Inferring --allCallOutputs=True to preserve probable actual outputs of a croo WDL file."
                     )
+                    options.all_call_outputs = True
+        
+            # This mutates document to add linting information, but doesn't print any lint errors itself
+            # or stop the workflow
+            WDL.Lint.lint(document)
+
+            # We use a mutable variable and a generic file pointer to capture information about lint warnings
+            # Both will be populated inside outline()
+            lint_warnings_counter = [0]
+            lint_warnings_io = io.StringIO()
+            outline(
+                document,
+                0,
+                file=lint_warnings_io,
+                show_called=(document.workflow is not None),
+                shown=lint_warnings_counter,
+            )  # type: ignore[no-untyped-call]
+
+            if getattr(WDL.Lint, "_shellcheck_available", None) is False:
+                logger.info("Suggestion: install shellcheck (www.shellcheck.net) to check task commands")
+
+            if lint_warnings_counter[0]:
+                logger.warning('Workflow lint warnings:\n%s', lint_warnings_io.getvalue().rstrip())
+                if options.strict:
+                    logger.critical(f'Workflow did not pass linting in strict mode')
+                    # MiniWDL uses exit code 2 to indicate linting errors, so replicate that behavior
+                    sys.exit(2)
+
+            # Get the execution directory
+            execution_dir = os.getcwd()
+
+            # Configure workflow interpreter options.
+            # TODO: Would be nice to somehow be able to change some of these on
+            # restart. For now we assume we are computing the same values.
+            wdl_options: WDLContext = {
+                "execution_dir": execution_dir,
+                "container": options.container,
+                "task_path": target.name,
+                "namespace": target.name,
+                "all_call_outputs": options.all_call_outputs,
+            }
+            assert wdl_options.get("container") is not None
 
-                    # But we can assume that we need to preserve individual
-                    # taks outputs since the point of CROO is fetching those
-                    # from Cromwell's output directories.
-                    #
-                    # This isn't quite WDL spec compliant but it will rescue
-                    # runs of the popular
-                    # <https://github.com/ENCODE-DCC/atac-seq-pipeline>
-                    if options.all_call_outputs is None:
-                        logger.warning(
-                            "Inferring --allCallOutputs=True to preserve probable actual outputs of a croo WDL file."
-                        )
-                        options.all_call_outputs = True
-
+            if options.restart:
+                output_bindings = toil.restart()
+            else:
                 # If our input really comes from a URI or path, remember it.
                 input_source_uri = None
                 # Also remember where we need to report JSON parse errors as
@@ -5592,19 +5686,6 @@ def main() -> None:
 
                 # TODO: Automatically set a good MINIWDL__SINGULARITY__IMAGE_CACHE ?
 
-                # Get the execution directory
-                execution_dir = os.getcwd()
-
-                # Configure workflow interpreter options
-                wdl_options: WDLContext = {
-                    "execution_dir": execution_dir,
-                    "container": options.container,
-                    "task_path": target.name,
-                    "namespace": target.name,
-                    "all_call_outputs": options.all_call_outputs,
-                }
-                assert wdl_options.get("container") is not None
-
                 # Run the workflow and get its outputs namespaced with the workflow name.
                 root_job = make_root_job(
                     target,