toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/wdl/wdltoil.py

@@ -62,13 +62,14 @@ else:
 
 from functools import partial
 from urllib.error import HTTPError
-from urllib.parse import quote, unquote, urljoin, urlsplit, urlparse
+from urllib.parse import quote, unquote, urljoin, urlsplit
 
 import WDL.Error
 import WDL.runtime.config
 from configargparse import ArgParser, Namespace
 from WDL._util import byte_size_units, chmod_R_plus
-from WDL.CLI import print_error
+from WDL.CLI import print_error, outline
+import WDL.Lint
 from WDL.runtime.backend.docker_swarm import SwarmContainer
 from WDL.runtime.backend.singularity import SingularityContainer
 from WDL.runtime.error import DownloadFailed
@@ -103,8 +104,8 @@ from toil.jobStores.abstractJobStore import (
 from toil.lib.exceptions import UnimplementedURLException
 from toil.lib.accelerators import get_individual_local_accelerators
 from toil.lib.conversions import VALID_PREFIXES, convert_units, human2bytes
+from toil.lib.trs import resolve_workflow
 from toil.lib.io import mkdtemp, is_any_url, is_file_url, TOIL_URI_SCHEME, is_standard_url, is_toil_url, is_remote_url
-from toil.lib.integration import resolve_workflow
 from toil.lib.memoize import memoize
 from toil.lib.misc import get_user_name
 from toil.lib.resources import ResourceMonitor
@@ -515,10 +516,14 @@ async def toil_read_source(
             # TODO: this is probably sync work that would be better as async work here
             AbstractJobStore.read_from_url(candidate_uri, destination_buffer)
         except Exception as e:
-            # TODO: we need to assume any error is just a not-found,
-            # because the exceptions thrown by read_from_url()
+            if isinstance(e, SyntaxError) or isinstance(e, NameError):
+                # These are probably actual problems with the code and not
+                # failures in reading the URL.
+                raise
+            # TODO: we need to assume in general that an error is just a
+            # not-found, because the exceptions thrown by read_from_url()
             # implementations are not specified.
-            logger.debug("Tried to fetch %s from %s but got %s", uri, candidate_uri, e)
+            logger.debug("Tried to fetch %s from %s but got %s: %s", uri, candidate_uri, type(e), e)
             continue
         # If we get here, we got it probably.
         try:
@@ -913,8 +918,8 @@ def set_shared_fs_path(file: WDL.Value.File, path: str) -> WDL.Value.File:
 
 
 def view_shared_fs_paths(
-    bindings: WDL.Env.Bindings[WDL.Value.Base],
-) -> WDL.Env.Bindings[WDL.Value.Base]:
+    bindings: WDLBindings,
+) -> WDLBindings:
     """
     Given WDL bindings, return a copy where all files have their shared filesystem paths as their values.
     """
@@ -1133,11 +1138,11 @@ def choose_human_readable_directory(
 
 def evaluate_decls_to_bindings(
     decls: list[WDL.Tree.Decl],
-    all_bindings: WDL.Env.Bindings[WDL.Value.Base],
+    all_bindings: WDLBindings,
     standard_library: ToilWDLStdLibBase,
     include_previous: bool = False,
     drop_missing_files: bool = False,
-) -> WDL.Env.Bindings[WDL.Value.Base]:
+) -> WDLBindings:
     """
     Evaluate decls with a given bindings environment and standard library.
     Creates a new bindings object that only contains the bindings from the given decls.
@@ -1152,7 +1157,7 @@ def evaluate_decls_to_bindings(
     """
     # all_bindings contains current bindings + previous all_bindings
     # bindings only contains the decl bindings themselves so that bindings from other sections prior aren't included
-    bindings: WDL.Env.Bindings[WDL.Value.Base] = WDL.Env.Bindings()
+    bindings: WDLBindings = WDL.Env.Bindings()
     drop_if_missing_with_workdir = partial(
         drop_if_missing, standard_library=standard_library
     )
@@ -1241,7 +1246,10 @@ class NonDownloadingSize(WDL.StdLib._Size):
         return WDL.Value.Float(total_size)
 
 
-def extract_workflow_inputs(environment: WDLBindings) -> list[str]:
+def extract_file_values(environment: WDLBindings) -> list[str]:
+    """
+    Get a list of all File object values in the given bindings.
+    """
     filenames = list()
 
     def add_filename(file: WDL.Value.File) -> WDL.Value.File:
@@ -1251,6 +1259,22 @@ def extract_workflow_inputs(environment: WDLBindings) -> list[str]:
     map_over_files_in_bindings(environment, add_filename)
     return filenames
 
+def extract_file_virtualized_values(environment: WDLBindings) -> list[str]:
+    """
+    Get a list of all File object virtualized values in the given bindings.
+
+    If a file hasn't been virtualized, it won't contribute to the list.
+    """
+    values = list()
+
+    def add_value(file: WDL.Value.File) -> WDL.Value.File:
+        value = get_file_virtualized_value(file)
+        if value is not None:
+            values.append(value)
+        return file
+
+    map_over_files_in_bindings(environment, add_value)
+    return values
 
 def convert_files(
     environment: WDLBindings,
@@ -1259,19 +1283,21 @@ def convert_files(
     task_path: str,
 ) -> WDLBindings:
     """
-    Resolve relative-URI files in the given environment convert the file values to a new value made from a given mapping.
-
-    Will return bindings with file values set to their corresponding relative-URI.
+    Fill in the virtualized_value fields for File objects in a WDL environment.
 
-    :param environment: Bindings to evaluate on
-    :return: new bindings object
+    :param environment: Bindings to evaluate on. Will not be modified.
+    :param file_to_id: Maps from imported URI to Toil FileID with the data.
+    :param file_to_data: Maps from WDL-level file calue to metadata about the
+        file, including URI that would have been imported.
+    :return: new bindings object with the annotated File objects in it.
     """
     dir_ids = {t[1] for t in file_to_data.values()}
     dir_to_id = {k: uuid.uuid4() for k in dir_ids}
 
     def convert_file_to_uri(file: WDL.Value.File) -> WDL.Value.File:
         """
-        Calls import_filename to detect if a potential URI exists and imports it. Will modify the File object value to the new URI and tack on the virtualized file.
+        Produce a WDL File with the virtualized_value set to the Toil URI for
+        the already-imported data, but the same value.
         """
         candidate_uri = file_to_data[file.value][0]
         file_id = file_to_id[candidate_uri]
@@ -1634,32 +1660,35 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             logger.debug("File has no virtualized value so not changing value")
         return file
 
+    def _resolve_devirtualized_to_uri(self, devirtualized: str) -> str:
+        """
+        Get a URI pointing to whatever URI or divirtualized file path is provided.
+
+        Handles resolving symlinks using in-container paths if necessary.
+        """
+        
+        return Toil.normalize_uri(devirtualized, dir_path=self.execution_dir)
+        
     def _virtualize_file(
         self, file: WDL.Value.File, enforce_existence: bool = True
     ) -> WDL.Value.File:
-        logger.debug("Virtualizing %s", file)
-        # If enforce_existence is true, then if a file is detected as nonexistent, raise an error. Else, let it pass through
         if get_file_virtualized_value(file) is not None:
-            logger.debug("File is marked nonexistent so passing it through")
+            # Already virtualized
             return file
 
-        if enforce_existence is False:
-            # We only want to error on a nonexistent file in the output section
-            # Since we need to virtualize on task boundaries, don't enforce existence if on a boundary
-            if is_standard_url(file.value):
-                file_uri = Toil.normalize_uri(file.value)
-            else:
-                abs_filepath = (
-                    os.path.join(self.execution_dir, file.value)
-                    if self.execution_dir is not None
-                    else os.path.abspath(file.value)
-                )
-                file_uri = Toil.normalize_uri(abs_filepath)
+        logger.debug("Virtualizing %s", file)
 
-            if not AbstractJobStore.url_exists(file_uri):
+        try:
+            # Let the actual virtualization implementation signal a missing file
+            virtualized_filename = self._virtualize_filename(file.value)
+        except FileNotFoundError:
+            if enforce_existence:
+                raise
+            else:
                 logger.debug("File appears nonexistent so marking it nonexistent")
+                # Mark the file nonexistent.
                 return set_file_nonexistent(file, True)
-        virtualized_filename = self._virtualize_filename(file.value)
+
         logger.debug(
             "For file %s got virtualized filename %s", file, virtualized_filename
         )
@@ -1842,9 +1871,12 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
     @memoize
     def _virtualize_filename(self, filename: str) -> str:
         """
-        from a local path in write_dir, 'virtualize' into the filename as it should present in a File value
+        from a local path or other URL, 'virtualize' into the filename as it should present in a File value.
+
+        New in Toil: the path or URL may not actually exist.
 
         :param filename: Can be a local file path, URL (http, https, s3, gs), or toilfile
+        :raises FileNotFoundError: if the file doesn't actually exist (new addition in Toil over MiniWDL)
         """
 
         if is_toil_url(filename):
@@ -1864,7 +1896,9 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             try:
                 imported = self._file_store.import_file(filename)
             except FileNotFoundError:
-                logger.error(
+                # This might happen because we're also along the code path for
+                # optional file outputs.
+                logger.info(
                     "File at URL %s does not exist or is inaccessible." % filename
                 )
                 raise
@@ -1875,9 +1909,13 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
                     filename,
                     e.code,
                 )
+                # We don't need to handle translating error codes for not
+                # found; import_file does it already. 
                 raise
             if imported is None:
-                # Satisfy mypy, this should never happen though as we don't pass a shared file name (which is the only way import_file returns None)
+                # Satisfy mypy. This should never happen though as we don't
+                # pass a shared file name (which is the only way import_file
+                # returns None)
                 raise RuntimeError("Failed to import URL %s into jobstore." % filename)
             file_basename = os.path.basename(urlsplit(filename).path)
             # Get the URL to the parent directory and use that.
@@ -1886,23 +1924,19 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
             dir_id = self._parent_dir_to_ids.setdefault(parent_dir, uuid.uuid4())
             result = pack_toil_uri(imported, self.task_path, dir_id, file_basename)
             logger.debug("Virtualized %s as WDL file %s", filename, result)
-            # We can't put the Toil URI in the virtualized_to_devirtualized cache because it would point to the URL instead of a
-            # local file on the machine, so only store the forward mapping
+            # We can't put the Toil URI in the virtualized_to_devirtualized
+            # cache because it would point to the URL instead of a local file
+            # on the machine, so only store the forward mapping
             self._devirtualized_to_virtualized[filename] = result
             return result
         else:
-            # Otherwise this is a local file and we want to fake it as a Toil file store file
-            # Make it an absolute path
-            parsed = urlparse(filename)
-            if parsed.scheme == "file":
-                # conversion was already done by normalize_uri
-                abs_filename = unquote(parsed.path)
-            elif self.execution_dir is not None:
-                # To support relative paths from execution directory, join the execution dir and filename
-                # If filename is already an abs path, join() will not do anything
-                abs_filename = os.path.join(self.execution_dir, filename)
-            else:
-                abs_filename = os.path.abspath(filename)
+            # Otherwise this is a local file name or URI and we want to fake it
+            # as a Toil file store file
+
+            # Convert to a properly-absolutized file URI
+            file_uri = Toil.normalize_uri(filename, dir_path=self.execution_dir)
+            # Extract the absolute path name
+            abs_filename = unquote(urlsplit(file_uri).path)
 
             if abs_filename in self._devirtualized_to_virtualized:
                 # This is a previously devirtualized thing so we can just use the
@@ -1913,6 +1947,9 @@ class ToilWDLStdLibBase(WDL.StdLib.Base):
                 )
                 return result
 
+            if not os.path.exists(abs_filename):
+                raise FileNotFoundError(abs_filename)
+
             file_id = self._file_store.writeGlobalFile(abs_filename)
 
             file_dir = os.path.dirname(abs_filename)
@@ -1942,6 +1979,51 @@ class ToilWDLStdLibWorkflow(ToilWDLStdLibBase):
 
         self._miniwdl_cache: Optional[WDL.runtime.cache.CallCache] = None
 
+    def _virtualize_file(
+        self, file: WDL.Value.File, enforce_existence: bool = True
+    ) -> WDL.Value.File:
+        # When a workflow coerces a string path or file: URI to a File at
+        # workflow scope, we need to fill in the cache filesystem path.
+        if (
+            get_file_virtualized_value(file) is None
+            and get_shared_fs_path(file) is None
+            and (
+                not is_any_url(file.value)
+                or is_file_url(file.value)
+            )
+        ):
+            # This is a never-virtualized file that is a file path or URI and
+            # has no shared FS path associated with it. We just made it at
+            # workflow scope. (If it came from a task, it would have a
+            # virtualized value already.)
+
+            # If we are loading it at workflow scope, the file path can be used
+            # as the cache path.
+
+            if not is_any_url(file.value):
+                # Handle file path
+                cache_path = file.value
+            else:
+                # Handle pulling path out of file URI
+                cache_path = unquote(urlsplit(file.value).path)
+
+            # Apply the path
+            file = set_shared_fs_path(file, cache_path)
+
+            logger.info(
+                "Applied shared filesystem path %s to File %s that appears to "
+                "have been coerced from String at workflow scope.",
+                cache_path,
+                file
+            )
+
+        # Do the virtualization
+        return super()._virtualize_file(file, enforce_existence)
+
+        # TODO: If the workflow coerces a File to a String and back again, we
+        # should have some way to recover the toilfile: URL it had in the job
+        # store to avoid re-importing it.
+
     # This needs to be hash-compatible with MiniWDL.
     # MiniWDL hooks _virtualize_filename
     # <https://github.com/chanzuckerberg/miniwdl/blob/475dd3f3784d1390e6a0e880d43316a620114de3/WDL/runtime/workflow.py#L699-L729>,
@@ -1995,7 +2077,7 @@ class ToilWDLStdLibWorkflow(ToilWDLStdLibBase):
             )
             # Make an environment of "file_sha256" to that as a WDL string, and
             # digest that, and make a write_ cache key. No need to transform to
-            # shared FS paths sonce no paths are in it.
+            # shared FS paths since no paths are in it.
             log_bindings(
                 logger.debug, "Digesting file bindings:", [file_input_bindings]
             )
@@ -2342,6 +2424,8 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
         filenames.
         """
 
+        logger.debug("WDL task outputs stdlib asked to virtualize %s", filename)
+
         if not is_any_url(filename) and not filename.startswith("/"):
             # We are getting a bare relative path on the supposedly devirtualized side.
             # Find a real path to it relative to the current directory override.
@@ -2390,8 +2474,12 @@ class ToilWDLStdLibTaskOutputs(ToilWDLStdLibBase, WDL.StdLib.TaskOutputs):
                 logger.error(
                     "Handling broken symlink %s ultimately to %s", filename, here
                 )
+                # This should produce a FileNotFoundError since we think of
+                # broken symlinks as nonexistent.
+                raise FileNotFoundError(filename)
             filename = here
-
+        
+        logger.debug("WDL task outputs stdlib thinks we really need to virtualize %s", filename)
         return super()._virtualize_filename(filename)
 
 
@@ -2531,7 +2619,7 @@ def devirtualize_files(
     that are actually available to command line commands.
     The same virtual file always maps to the same devirtualized filename even with duplicates
     """
-    logger.info("Devirtualizing files")
+    logger.debug("Devirtualizing files")
     return map_over_files_in_bindings(environment, stdlib._devirtualize_file)
 
 
@@ -2542,12 +2630,35 @@ def virtualize_files(
     Make sure all the File values embedded in the given bindings point to files
     that are usable from other machines.
     """
-    logger.info("Virtualizing files")
+    logger.debug("Virtualizing files")
     virtualize_func = partial(
         stdlib._virtualize_file, enforce_existence=enforce_existence
     )
     return map_over_files_in_bindings(environment, virtualize_func)
 
+def delete_dead_files(internal_bindings: WDLBindings, live_bindings_list: list[WDLBindings], file_store: AbstractFileStore) -> None:
+    """
+    Delete any files that in the given bindings but not in the live list.
+
+    Operates on the virtualized values of File objects anywhere in the bindings.
+    """
+
+    # Get all the files in the first bindings and not any of the others.
+    unused_files = set(
+        extract_file_virtualized_values(internal_bindings)
+    ).difference(
+        *(
+            extract_file_virtualized_values(bindings)
+            for bindings in live_bindings_list
+        )
+    )
+
+    for file_uri in unused_files:
+        # Delete them
+        if is_toil_url(file_uri):
+            logger.debug("Delete file %s that is not needed", file_uri)
+            file_id, _, _, _ = unpack_toil_uri(file_uri)
+            file_store.deleteGlobalFile(file_id)
 
 def add_paths(task_container: TaskContainer, host_paths: Iterable[str]) -> None:
     """
@@ -3021,6 +3132,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
         self,
         task: WDL.Tree.Task,
         prev_node_results: Sequence[Promised[WDLBindings]],
+        enclosing_bindings: WDLBindings,
         task_id: list[str],
         wdl_options: WDLContext,
         **kwargs: Any,
@@ -3028,6 +3140,11 @@ class WDLTaskWrapperJob(WDLBaseJob):
         """
         Make a new job to determine resources and run a task.
 
+        :param enclosing_bindings: Bindings in the enclosing section,
+            containing files not to clean up. Files that are passed as inputs
+            but not uses as outputs or present in the enclosing section
+            bindings will be deleted after the task call completes.
+
         :param namespace: The namespace that the task's *contents* exist in.
                The caller has alredy added the task's own name.
         """
@@ -3048,6 +3165,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
 
         self._task = task
         self._prev_node_results = prev_node_results
+        self._enclosing_bindings = enclosing_bindings
         self._task_id = task_id
 
     @report_wdl_errors("evaluate task code", exit=True)
@@ -3087,10 +3205,23 @@ class WDLTaskWrapperJob(WDLBaseJob):
             # TODO: What if the same file is passed through several tasks, and
             # we get cache hits on those tasks? Won't we upload it several
             # times?
+
+            # Load output bindings from the cache
+            cached_bindings = virtualize_files(
+                cached_result, standard_library, enforce_existence=False
+            )
+
+            # Throw away anything input but not available outside the call or
+            # output.
+            delete_dead_files(
+                bindings,
+                [cached_bindings, self._enclosing_bindings],
+                file_store
+            )
+
+            # Postprocess and ship the output bindings.
             return self.postprocess(
-                virtualize_files(
-                    cached_result, standard_library, enforce_existence=False
-                )
+                cached_bindings
             )
 
         if self._task.inputs:
@@ -3227,6 +3358,7 @@ class WDLTaskWrapperJob(WDLBaseJob):
             virtualize_files(
                 runtime_bindings, standard_library, enforce_existence=False
             ),
+            self._enclosing_bindings,
             self._task_id,
             cores=runtime_cores or self.cores,
             memory=runtime_memory or self.memory,
@@ -3262,6 +3394,7 @@ class WDLTaskJob(WDLBaseJob):
         task: WDL.Tree.Task,
         task_internal_bindings: Promised[WDLBindings],
         runtime_bindings: Promised[WDLBindings],
+        enclosing_bindings: WDLBindings,
         task_id: list[str],
         mount_spec: dict[str | None, int],
         wdl_options: WDLContext,
@@ -3271,6 +3404,9 @@ class WDLTaskJob(WDLBaseJob):
         """
         Make a new job to run a task.
 
+        :param enclosing_bindings: Bindings outside the workflow call, with
+            files that should not be cleaned up at the end of the task.
+
         :param namespace: The namespace that the task's *contents* exist in.
                The caller has alredy added the task's own name.
         """
@@ -3294,6 +3430,7 @@ class WDLTaskJob(WDLBaseJob):
         self._task = task
         self._task_internal_bindings = task_internal_bindings
         self._runtime_bindings = runtime_bindings
+        self._enclosing_bindings = enclosing_bindings
         self._task_id = task_id
         self._cache_key = cache_key
         self._mount_spec = mount_spec
@@ -4052,6 +4189,18 @@ class WDLTaskJob(WDLBaseJob):
                 miniwdl_config=miniwdl_config,
             )
 
+        # Clean up anything from the task call input: block or the runtime
+        # section that isn't getting output or available in the enclosing
+        # section. Runtime sections aren't meant to have files, but nothing
+        # actually stops them from being there.
+        delete_dead_files(
+            combine_bindings([bindings, runtime_bindings]),
+            [output_bindings, self._enclosing_bindings],
+            file_store
+        )
+        # If File objects somehow made it to the runtime block they shouldn't
+        # have been virtualized so don't bother with them.
+
         # Do postprocessing steps to e.g. apply namespaces.
         output_bindings = self.postprocess(output_bindings)
 
@@ -4104,7 +4253,8 @@ class WDLWorkflowNodeJob(WDLBaseJob):
             logger.info("Setting %s to %s", self._node.name, self._node.expr)
             value = evaluate_decl(self._node, incoming_bindings, standard_library)
             bindings = incoming_bindings.bind(self._node.name, value)
-            return self.postprocess(bindings)
+            # TODO: Only virtualize the new binding
+            return self.postprocess(virtualize_files(bindings, standard_library, enforce_existence=False))
         elif isinstance(self._node, WDL.Tree.Call):
             # This is a call of a task or workflow
 
@@ -4125,6 +4275,8 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 standard_library,
                 inputs_mapping,
             )
+            # Prepare call inputs to move to another node
+            input_bindings = virtualize_files(input_bindings, standard_library, enforce_existence=False)
 
             # Bindings may also be added in from the enclosing workflow inputs
             # TODO: this is letting us also inject them from the workflow body.
@@ -4142,6 +4294,7 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 subjob: WDLBaseJob = WDLWorkflowJob(
                     self._node.callee,
                     [input_bindings, passed_down_bindings],
+                    incoming_bindings,
                     self._node.callee_id,
                     wdl_options=wdl_options,
                     local=True,
@@ -4152,6 +4305,7 @@ class WDLWorkflowNodeJob(WDLBaseJob):
                 subjob = WDLTaskWrapperJob(
                     self._node.callee,
                     [input_bindings, passed_down_bindings],
+                    incoming_bindings,
                     self._node.callee_id,
                     wdl_options=wdl_options,
                     local=True,
@@ -4253,7 +4407,8 @@ class WDLWorkflowNodeListJob(WDLBaseJob):
                     node, "Unimplemented WorkflowNode: " + str(type(node))
                 )
 
-        return self.postprocess(current_bindings)
+        # TODO: Only virtualize the new bindings created
+        return self.postprocess(virtualize_files(current_bindings, standard_library, enforce_existence=False))
 
 
 class WDLCombineBindingsJob(WDLBaseJob):
@@ -5016,6 +5171,7 @@ class WDLWorkflowJob(WDLSectionJob):
         self,
         workflow: WDL.Tree.Workflow,
         prev_node_results: Sequence[Promised[WDLBindings]],
+        enclosing_bindings: WDLBindings,
         workflow_id: list[str],
         wdl_options: WDLContext,
         **kwargs: Any,
@@ -5024,6 +5180,13 @@ class WDLWorkflowJob(WDLSectionJob):
         Create a subtree that will run a WDL workflow. The job returns the
         return value of the workflow.
 
+        :param prev_node_results: Bindings fed into the workflow call as inputs.
+
+        :param enclosing_bindings: Bindings in the enclosing section,
+            containing files not to clean up. Files that are passed as inputs
+            but not uses as outputs or present in the enclosing section
+            bindings will be deleted after the workflow call completes.
+
         :param namespace: the namespace that the workflow's *contents* will be
                in. Caller has already added the workflow's own name.
         """
@@ -5040,6 +5203,7 @@ class WDLWorkflowJob(WDLSectionJob):
 
         self._workflow = workflow
         self._prev_node_results = prev_node_results
+        self._enclosing_bindings = enclosing_bindings
         self._workflow_id = workflow_id
 
     @report_wdl_errors("run workflow")
@@ -5091,11 +5255,13 @@ class WDLWorkflowJob(WDLSectionJob):
         # Make jobs to run all the parts of the workflow
         sink = self.create_subgraph(self._workflow.body, [], bindings)
 
-        # To support the all call outputs feature, run an outputs job even if
-        # we have a declared but empty outputs section.
+        # To support the all call outputs feature and cleanup of files created
+        # in input: blocks, run an outputs job even if we have a declared but
+        # empty outputs section.
         outputs_job = WDLOutputsJob(
             self._workflow,
             sink.rv(),
+            self._enclosing_bindings,
             wdl_options=self._wdl_options,
             cache_key=cache_key,
             local=True,
@@ -5117,6 +5283,7 @@ class WDLOutputsJob(WDLBaseJob):
         self,
         workflow: WDL.Tree.Workflow,
         bindings: Promised[WDLBindings],
+        enclosing_bindings: WDLBindings,
         wdl_options: WDLContext,
         cache_key: str | None = None,
         **kwargs: Any,
@@ -5124,6 +5291,11 @@ class WDLOutputsJob(WDLBaseJob):
         """
         Make a new WDLWorkflowOutputsJob for the given workflow, with the given set of bindings after its body runs.
 
+        :param bindings: Bindings after execution of the workflow body.
+
+        :param enclosing_bindings: Bindings outside the workflow call, with
+            files that should not be cleaned up at the end of the workflow.
+
         :param cache_key: If set and storing into the call cache is on, will
                cache the workflow execution result under the given key in a
                MiniWDL-compatible way.
@@ -5131,6 +5303,7 @@ class WDLOutputsJob(WDLBaseJob):
         super().__init__(wdl_options=wdl_options, **kwargs)
 
         self._bindings = bindings
+        self._enclosing_bindings = enclosing_bindings
         self._workflow = workflow
         self._cache_key = cache_key
 
@@ -5223,8 +5396,15 @@ class WDLOutputsJob(WDLBaseJob):
                 self._cache_key, output_bindings, file_store, self._wdl_options
             )
 
-        return self.postprocess(output_bindings)
+        # Let Files that are not output or available outside the call go out of
+        # scope.
+        delete_dead_files(
+            unwrap(self._bindings),
+            [output_bindings, self._enclosing_bindings],
+            file_store
+        )
 
+        return self.postprocess(output_bindings)
 
 class WDLStartJob(WDLSectionJob):
     """
@@ -5259,18 +5439,24 @@ class WDLStartJob(WDLSectionJob):
         if isinstance(self._target, WDL.Tree.Workflow):
             # Create a workflow job. We rely in this to handle entering the input
             # namespace if needed, or handling free-floating inputs.
+            # Pass top-level inputs as enclosing section inputs to avoid
+            # bothering to separately delete them.
             job: WDLBaseJob = WDLWorkflowJob(
                 self._target,
                 [inputs],
+                inputs,
                 [self._target.name],
                 wdl_options=self._wdl_options,
                 local=True,
             )
         else:
             # There is no workflow. Create a task job.
+            # Pass top-level inputs as enclosing section inputs to avoid
+            # bothering to separately delete them.
             job = WDLTaskWrapperJob(
                 self._target,
                 [inputs],
+                inputs,
                 [self._target.name],
                 wdl_options=self._wdl_options,
                 local=True,
@@ -5344,7 +5530,7 @@ class WDLImportWrapper(WDLSectionJob):
         self._import_workers_disk = import_workers_disk
 
     def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
-        filenames = extract_workflow_inputs(self._inputs)
+        filenames = extract_file_values(self._inputs)
         file_to_data = get_file_sizes(
             filenames,
             file_store.jobStore,
@@ -5438,56 +5624,108 @@ def main() -> None:
     )
 
     try:
-        with Toil(options) as toil:
-            if options.restart:
-                output_bindings = toil.restart()
+        wdl_uri, trs_spec = resolve_workflow(options.wdl_uri, supported_languages={"WDL"})
+
+        with Toil(options, workflow_name=trs_spec or wdl_uri, trs_spec=trs_spec) as toil:
+            # TODO: Move all the input parsing outside the Toil context
+            # manager to avoid leaving a job store behind if the workflow
+            # can't start.
+
+            # Both start and restart need us to have the workflow and the
+            # wdl_options WDLContext.
+
+            # MiniWDL load code internally uses asyncio.get_event_loop()
+            # which might not get an event loop if somebody has ever called
+            # set_event_loop. So we need to make sure an event loop is
+            # available.
+            asyncio.set_event_loop(asyncio.new_event_loop())
+
+            # Load the WDL document.
+            document: WDL.Tree.Document = WDL.load(
+                wdl_uri,
+                read_source=toil_read_source,
+            )
+
+            # See if we're going to run a workflow or a task
+            target: WDL.Tree.Workflow | WDL.Tree.Task
+            if document.workflow:
+                target = document.workflow
+            elif len(document.tasks) == 1:
+                target = document.tasks[0]
+            elif len(document.tasks) > 1:
+                raise WDL.Error.InputError(
+                    "Multiple tasks found with no workflow! Either add a workflow or keep one task."
+                )
             else:
-                # TODO: Move all the input parsing outside the Toil context
-                # manager to avoid leaving a job store behind if the workflow
-                # can't start.
-
-                # Load the WDL document
-                document: WDL.Tree.Document = WDL.load(
-                    resolve_workflow(options.wdl_uri, supported_languages={"WDL"}),
-                    read_source=toil_read_source,
+                raise WDL.Error.InputError("WDL document is empty!")
+
+            if "croo_out_def" in target.meta:
+                # This workflow or task wants to have its outputs
+                # "organized" by the Cromwell Output Organizer:
+                # <https://github.com/ENCODE-DCC/croo>.
+                #
+                # TODO: We don't support generating anything that CROO can read.
+                logger.warning(
+                    "This WDL expects to be used with the Cromwell Output Organizer (croo) <https://github.com/ENCODE-DCC/croo>. Toil cannot yet produce the outputs that croo requires. You will not be able to use croo on the output of this Toil run!"
                 )
 
-                # See if we're going to run a workflow or a task
-                target: WDL.Tree.Workflow | WDL.Tree.Task
-                if document.workflow:
-                    target = document.workflow
-                elif len(document.tasks) == 1:
-                    target = document.tasks[0]
-                elif len(document.tasks) > 1:
-                    raise WDL.Error.InputError(
-                        "Multiple tasks found with no workflow! Either add a workflow or keep one task."
-                    )
-                else:
-                    raise WDL.Error.InputError("WDL document is empty!")
-
-                if "croo_out_def" in target.meta:
-                    # This workflow or task wants to have its outputs
-                    # "organized" by the Cromwell Output Organizer:
-                    # <https://github.com/ENCODE-DCC/croo>.
-                    #
-                    # TODO: We don't support generating anything that CROO can read.
+                # But we can assume that we need to preserve individual
+                # taks outputs since the point of CROO is fetching those
+                # from Cromwell's output directories.
+                #
+                # This isn't quite WDL spec compliant but it will rescue
+                # runs of the popular
+                # <https://github.com/ENCODE-DCC/atac-seq-pipeline>
+                if options.all_call_outputs is None:
                     logger.warning(
-                        "This WDL expects to be used with the Cromwell Output Organizer (croo) <https://github.com/ENCODE-DCC/croo>. Toil cannot yet produce the outputs that croo requires. You will not be able to use croo on the output of this Toil run!"
+                        "Inferring --allCallOutputs=True to preserve probable actual outputs of a croo WDL file."
                     )
+                    options.all_call_outputs = True
+        
+            # This mutates document to add linting information, but doesn't print any lint errors itself
+            # or stop the workflow
+            WDL.Lint.lint(document)
+
+            # We use a mutable variable and a generic file pointer to capture information about lint warnings
+            # Both will be populated inside outline()
+            lint_warnings_counter = [0]
+            lint_warnings_io = io.StringIO()
+            outline(
+                document,
+                0,
+                file=lint_warnings_io,
+                show_called=(document.workflow is not None),
+                shown=lint_warnings_counter,
+            )  # type: ignore[no-untyped-call]
+
+            if getattr(WDL.Lint, "_shellcheck_available", None) is False:
+                logger.info("Suggestion: install shellcheck (www.shellcheck.net) to check task commands")
+
+            if lint_warnings_counter[0]:
+                logger.warning('Workflow lint warnings:\n%s', lint_warnings_io.getvalue().rstrip())
+                if options.strict:
+                    logger.critical(f'Workflow did not pass linting in strict mode')
+                    # MiniWDL uses exit code 2 to indicate linting errors, so replicate that behavior
+                    sys.exit(2)
+
+            # Get the execution directory
+            execution_dir = os.getcwd()
+
+            # Configure workflow interpreter options.
+            # TODO: Would be nice to somehow be able to change some of these on
+            # restart. For now we assume we are computing the same values.
+            wdl_options: WDLContext = {
+                "execution_dir": execution_dir,
+                "container": options.container,
+                "task_path": target.name,
+                "namespace": target.name,
+                "all_call_outputs": options.all_call_outputs,
+            }
+            assert wdl_options.get("container") is not None
 
-                    # But we can assume that we need to preserve individual
-                    # taks outputs since the point of CROO is fetching those
-                    # from Cromwell's output directories.
-                    #
-                    # This isn't quite WDL spec compliant but it will rescue
-                    # runs of the popular
-                    # <https://github.com/ENCODE-DCC/atac-seq-pipeline>
-                    if options.all_call_outputs is None:
-                        logger.warning(
-                            "Inferring --allCallOutputs=True to preserve probable actual outputs of a croo WDL file."
-                        )
-                        options.all_call_outputs = True
-
+            if options.restart:
+                output_bindings = toil.restart()
+            else:
                 # If our input really comes from a URI or path, remember it.
                 input_source_uri = None
                 # Also remember where we need to report JSON parse errors as
@@ -5564,12 +5802,14 @@ def main() -> None:
                     inputs_search_path.append(input_source_uri)
 
                     match = re.match(
-                        r"https://raw\.githubusercontent\.com/[^/]*/[^/]*/[^/]*/",
+                        r"https://raw\.githubusercontent\.com/[^/]*/[^/]*/(refs/heads/)?[^/]*/",
                         input_source_uri,
                     )
                     if match:
                         # Special magic for Github repos to make e.g.
                         # https://raw.githubusercontent.com/vgteam/vg_wdl/44a03d9664db3f6d041a2f4a69bbc4f65c79533f/params/giraffe.json
+                        # or
+                        # https://raw.githubusercontent.com/vgteam/vg_wdl/refs/heads/giraffedv/params/giraffe.json
                         # work when it references things relative to repo root.
                         logger.info(
                             "Inputs appear to come from a Github repository; adding repository root to file search path"
@@ -5578,19 +5818,6 @@ def main() -> None:
 
                 # TODO: Automatically set a good MINIWDL__SINGULARITY__IMAGE_CACHE ?
 
-                # Get the execution directory
-                execution_dir = os.getcwd()
-
-                # Configure workflow interpreter options
-                wdl_options: WDLContext = {
-                    "execution_dir": execution_dir,
-                    "container": options.container,
-                    "task_path": target.name,
-                    "namespace": target.name,
-                    "all_call_outputs": options.all_call_outputs,
-                }
-                assert wdl_options.get("container") is not None
-
                 # Run the workflow and get its outputs namespaced with the workflow name.
                 root_job = make_root_job(
                     target,