fractal-server 2.14.0a15__py3-none-any.whl → 2.14.0a17__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a15"
+__VERSION__ = "2.14.0a17"

fractal_server/app/runner/executors/local/runner.py

@@ -11,6 +11,9 @@ from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -69,7 +72,12 @@ class LocalRunner(BaseRunner):
         workdir_local = task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
+        # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # SUBMISSION PHASE
         future = self.executor.submit(
@@ -155,17 +163,35 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                current_task_files = list_task_files[positional_index]
-                current_task_files.prefix = (
-                    f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
-                )
+                list_task_files[
+                    positional_index
+                ].prefix = f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
                 future = self.executor.submit(
                     func,
                     parameters=kwargs,
-                    remote_files=current_task_files.remote_files_dict,
+                    remote_files=list_task_files[
+                        positional_index
+                    ].remote_files_dict,
                 )
                 active_futures[positional_index] = future
 
+                if task_type == "parallel":
+                    # FIXME: replace loop with a `bulk_update_history_unit`
+                    # function
+                    update_logfile_of_history_unit(
+                        history_unit_id=history_unit_ids[positional_index],
+                        logfile=list_task_files[
+                            positional_index
+                        ].log_file_local,
+                    )
+                else:
+                    logger.debug(
+                        f"Unclear what logfile to associate to {task_type=} "
+                        "within multisubmit (see issue #2382)."
+                    )
+                    # FIXME: Improve definition for compound tasks
+                    pass
+
             while active_futures:
                 # FIXME: add shutdown detection
                 # if file exists: cancel all futures, and raise

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,12 +23,18 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
+SHUTDOWN_ERROR_MESSAGE = "Failed due to job-execution shutdown."
+SHUTDOWN_EXCEPTION = JobExecutionError(SHUTDOWN_ERROR_MESSAGE)
+
 logger = set_logger(__name__)
 
 # FIXME: Transform several logger.info into logger.debug.
@@ -91,6 +97,7 @@ class BaseSlurmRunner(BaseRunner):
 
     def run_squeue(self, job_ids: list[str]) -> tuple[bool, str]:
         # FIXME: review different cases (exception vs no job found)
+        # FIXME: Fail for empty list
         job_id_single_str = ",".join([str(j) for j in job_ids])
         cmd = (
             f"squeue --noheader --format='%i %T' --jobs {job_id_single_str}"
@@ -225,9 +232,11 @@ class BaseSlurmRunner(BaseRunner):
         logger.info(script_lines)
 
         # Always print output of `uname -n` and `pwd`
-        script_lines.append("Hostname: $(uname -n)\n")
-        script_lines.append("Current directory : $(pwd)\n")
-        script_lines.append('Start time: $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
+        script_lines.append('\necho "Hostname: $(uname -n)"')
+        script_lines.append('echo "Current directory : $(pwd)"')
+        script_lines.append(
+            'echo "Start time: $(date +"%Y-%m-%dT%H:%M:%S%z")"'
+        )
 
         # Complete script preamble
         script_lines.append("\n")
@@ -242,7 +251,9 @@ class BaseSlurmRunner(BaseRunner):
             )
         script_lines.append("wait\n")
         script = "\n".join(script_lines)
-        script_lines.append('End time:   $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
+        script_lines.append(
+            'echo "End time:   $(date +"%Y-%m-%dT%H:%M:%S%z")"'
+        )
 
         # Write submission script
         with open(slurm_job.slurm_submission_script_local, "w") as f:
@@ -332,7 +343,10 @@ class BaseSlurmRunner(BaseRunner):
             pass
 
     def _postprocess_single_task(
-        self, *, task: SlurmTask
+        self,
+        *,
+        task: SlurmTask,
+        was_job_scancelled: bool = False,
     ) -> tuple[Any, Exception]:
         try:
             with open(task.output_pickle_file_local, "rb") as f:
@@ -344,17 +358,24 @@ class BaseSlurmRunner(BaseRunner):
             else:
                 exception = _handle_exception_proxy(output)
                 return None, exception
+
         except Exception as e:
             exception = JobExecutionError(f"ERROR, {str(e)}")
+            # If job was scancelled and task failed, replace
+            # exception with a shutdown-related one.
+            if was_job_scancelled:
+                logger.debug(
+                    "Replacing exception with a shutdown-related one, "
+                    f"for {task.index=}."
+                )
+                exception = SHUTDOWN_EXCEPTION
+
             return None, exception
         finally:
-            pass
-            # FIXME: Re-include unlinks of pickle files
-            # Path(task.input_pickle_file_local).unlink(missing_ok=True)
-            # Path(task.output_pickle_file_local).unlink(missing_ok=True)
+            Path(task.input_pickle_file_local).unlink(missing_ok=True)
+            Path(task.output_pickle_file_local).unlink(missing_ok=True)
 
     def is_shutdown(self) -> bool:
-        # FIXME: shutdown is not implemented
         return self.shutdown_file.exists()
 
     @property
@@ -385,7 +406,14 @@ class BaseSlurmRunner(BaseRunner):
             raise JobExecutionError("Unexpected branch: jobs should be empty.")
 
         if self.is_shutdown():
-            raise JobExecutionError("Cannot continue after shutdown.")
+            with next(get_sync_db()) as db:
+                update_status_of_history_unit(
+                    history_unit_id=history_unit_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
+
+            return None, SHUTDOWN_EXCEPTION
 
         # Validation phase
         self.validate_submit_parameters(
@@ -401,6 +429,10 @@ class BaseSlurmRunner(BaseRunner):
 
         # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # Submission phase
         slurm_job = SlurmJob(
@@ -437,19 +469,29 @@ class BaseSlurmRunner(BaseRunner):
         # Retrieval phase
         logger.info("[submit] START retrieval phase")
         while len(self.jobs) > 0:
+
+            # Handle shutdown
+            scancelled_job_ids = []
             if self.is_shutdown():
-                self.scancel_jobs()
+                logger.info("[submit] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[submit] {scancelled_job_ids=}")
+
+            # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
-            logger.info(f"{finished_job_ids=}")
+            logger.debug(f"[submit] {finished_job_ids=}")
+
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
-                    logger.info(f"Now process {slurm_job_id=}")
+                    logger.debug(f"[submit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
-
                     self._copy_files_from_remote_to_local(slurm_job)
+                    was_job_scancelled = slurm_job_id in scancelled_job_ids
                     result, exception = self._postprocess_single_task(
-                        task=slurm_job.tasks[0]
+                        task=slurm_job.tasks[0],
+                        was_job_scancelled=was_job_scancelled,
                     )
+
                     if exception is not None:
                         update_status_of_history_unit(
                             history_unit_id=history_unit_id,
@@ -477,13 +519,29 @@ class BaseSlurmRunner(BaseRunner):
         list_task_files: list[TaskFiles],
         task_type: Literal["parallel", "compound", "converter_compound"],
         config: SlurmConfig,
-    ):
+    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
 
         if len(self.jobs) > 0:
             raise RuntimeError(
-                f"Cannot run .multisubmit when {len(self.jobs)=}"
+                f"Cannot run `multisubmit` when {len(self.jobs)=}"
             )
 
+        if self.is_shutdown():
+            if task_type == "parallel":
+                with next(get_sync_db()) as db:
+                    # FIXME: Replace with bulk function
+                    for history_unit_id in history_unit_ids:
+                        update_status_of_history_unit(
+                            history_unit_id=history_unit_id,
+                            status=HistoryUnitStatus.FAILED,
+                            db_sync=db,
+                        )
+            results = {}
+            exceptions = {
+                ind: SHUTDOWN_EXCEPTION for ind in range(len(list_parameters))
+            }
+            return results, exceptions
+
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
             task_type=task_type,
@@ -550,18 +608,17 @@ class BaseSlurmRunner(BaseRunner):
             tasks = []
             for ind_chunk, parameters in enumerate(chunk):
                 index = (ind_batch * batch_size) + ind_chunk
-                current_task_files = list_task_files[index]
-                current_task_files.prefix = prefix
+                list_task_files[index].prefix = prefix
                 tasks.append(
                     SlurmTask(
                         prefix=prefix,
                         index=index,
-                        component=current_task_files.component,
+                        component=list_task_files[index].component,
                         workdir_local=workdir_local,
                         workdir_remote=workdir_remote,
                         parameters=parameters,
                         zarr_url=parameters["zarr_url"],
-                        task_files=current_task_files,
+                        task_files=list_task_files[index],
                     ),
                 )
 
@@ -576,6 +633,21 @@ class BaseSlurmRunner(BaseRunner):
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
+        if task_type == "parallel":
+            # FIXME: replace loop with a `bulk_update_history_unit` function
+            for ind, task_files in enumerate(list_task_files):
+                update_logfile_of_history_unit(
+                    history_unit_id=history_unit_ids[ind],
+                    logfile=task_files.log_file_local,
+                )
+        else:
+            logger.debug(
+                f"Unclear what logfile to associate to {task_type=} "
+                "within multisubmit (see issue #2382)."
+            )
+            # FIXME: Improve definition for compound tasks
+            pass
+
         logger.info(f"END submission phase, {self.job_ids=}")
 
         # FIXME: replace this sleep a more precise check
@@ -584,22 +656,34 @@ class BaseSlurmRunner(BaseRunner):
         logger.warning(f"[submit] Now sleep {sleep_time} (FIXME)")
         time.sleep(sleep_time)
 
+        # FIXME: Could we merge the submit/multisubmit retrieval phases?
+
         # Retrieval phase
-        logger.info("START retrieval phase")
+        logger.info("[multisubmit] START retrieval phase")
         while len(self.jobs) > 0:
+
+            # Handle shutdown
+            scancelled_job_ids = []
             if self.is_shutdown():
-                self.scancel_jobs()
+                logger.info("[multisubmit] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[multisubmit] {scancelled_job_ids=}")
+
+            # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
-            logger.info(f"{finished_job_ids=}")
+            logger.debug(f"[multisubmit] {finished_job_ids=}")
+
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
-                    logger.info(f"Now processing {slurm_job_id=}")
+                    logger.info(f"[multisubmit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
                     self._copy_files_from_remote_to_local(slurm_job)
                     for task in slurm_job.tasks:
-                        logger.info(f"Now processing {task.index=}")
+                        logger.info(f"[multisubmit] Now process {task.index=}")
+                        was_job_scancelled = slurm_job_id in scancelled_job_ids
                         result, exception = self._postprocess_single_task(
-                            task=task
+                            task=task,
+                            was_job_scancelled=was_job_scancelled,
                         )
 
                         # Note: the relevant done/failed check is based on
@@ -659,11 +743,12 @@ class BaseSlurmRunner(BaseRunner):
             logger.error(error_msg)
             raise RuntimeError(error_msg)
 
-    def scancel_jobs(self) -> None:
+    def scancel_jobs(self) -> list[str]:
         logger.info("[scancel_jobs] START")
 
         if self.jobs:
-            scancel_string = " ".join(self.job_ids)
+            scancelled_job_ids = self.job_ids
+            scancel_string = " ".join(scancelled_job_ids)
             scancel_cmd = f"scancel {scancel_string}"
             logger.warning(f"Now scancel-ing SLURM jobs {scancel_string}")
             try:
@@ -673,5 +758,5 @@ class BaseSlurmRunner(BaseRunner):
                     "[scancel_jobs] `scancel` command failed. "
                     f"Original error:\n{str(e)}"
                 )
-
         logger.info("[scancel_jobs] END")
+        return scancelled_job_ids

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -140,7 +140,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
                     f"Original error: {str(e)}"
                 )
 
-    def _run_remote_cmd(self, cmd: str):
+    def _run_remote_cmd(self, cmd: str) -> str:
         res = _run_command_as_user(
             cmd=cmd,
             user=self.slurm_user,
@@ -149,6 +149,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
         )
         return res.stdout
 
-    def _run_local_cmd(self, cmd: str):
+    def _run_local_cmd(self, cmd: str) -> str:
         res = _subprocess_run_or_raise(cmd)
         return res.stdout

fractal_server/app/runner/v2/db_tools.py

@@ -3,11 +3,14 @@ from typing import Any
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 
+_CHUNK_SIZE = 2_000
+
 
 def update_status_of_history_run(
     *,
@@ -37,7 +40,18 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
-_CHUNK_SIZE = 2_000
+def update_logfile_of_history_unit(
+    *,
+    history_unit_id: int,
+    logfile: str,
+) -> None:
+    with next(get_sync_db()) as db_sync:
+        unit = db_sync.get(HistoryUnit, history_unit_id)
+        if unit is None:
+            raise ValueError(f"HistoryUnit {history_unit_id} not found.")
+        unit.logfile = logfile
+        db_sync.merge(unit)
+        db_sync.commit()
 
 
 def bulk_upsert_image_cache_fast(

fractal_server/app/runner/v2/runner_functions.py

@@ -172,7 +172,7 @@ def run_v2_task_non_parallel(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=task_files.log_file_local,
+            logfile=None,
             zarr_urls=zarr_urls,
         )
         db.add(history_unit)
@@ -275,7 +275,7 @@ def run_v2_task_parallel(
         HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=list_task_files[ind].log_file_local,
+            logfile=None,
             zarr_urls=[image["zarr_url"]],
         )
         for ind, image in enumerate(images)
@@ -401,8 +401,7 @@ def run_v2_task_compound(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            # FIXME: What about compute-task logs?
-            logfile=task_files_init.log_file_local,
+            logfile=None,
             zarr_urls=input_image_zarr_urls,
         )
         db.add(history_unit)

{fractal_server-2.14.0a15.dist-info → fractal_server-2.14.0a17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a15
+Version: 2.14.0a17
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.0a15.dist-info → fractal_server-2.14.0a17.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=RTgktqJdR_QVcUPwuSBCYyBhAT7xsBxkixaJXZsTluc,26
+fractal_server/__init__.py,sha256=Kv1hRPs4otMnorymxtQd2kL3M7DCvXZqfxY3-m9qJ3U,26
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -73,13 +73,13 @@ fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 fractal_server/app/runner/executors/base_runner.py,sha256=s5aZLDPzC565FadaqFxrCLIlQzBn2D9iOpEjnBZROkk,5541
 fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/local/get_local_config.py,sha256=wbrIYuGOvABOStrE7jNrC4ULPhtBQ5Q7Y3aKm_icomg,3508
-fractal_server/app/runner/executors/local/runner.py,sha256=o_s8DIwI0UlBQ8NozPLG9VCtvBkhlCkC7um7yAei2j4,7936
+fractal_server/app/runner/executors/local/runner.py,sha256=Kv_ZTRARAg-lAhh-4tbSE1JcwukqGzgeQk0RAMLFgGk,8963
 fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
 fractal_server/app/runner/executors/slurm_common/_handle_exception_proxy.py,sha256=jU2N4vMafdcDPqVXwSApu4zxskCqhHmsXF3hBpOAAFA,577
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=fZaFUUXqDH0p3DndCFUpFqTqyD2tMVCuSYgYLAycpVw,15897
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=y9FDJtgqLT8BA6vaxdpM-49bzNMOeTdul4zZJkv288A,25832
+fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=4U3ZAN6QbBd_ukNapU8DHQ6Qughf7HmfNdi9Q2Nmd6g,29132
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=-fAX1DZMB5RZnyYanIJD72mWOJAPkh21jd4loDXKJw4,5994
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=iXLu4d-bWzn7qmDaOjKFkcuaSHLjPESAMSLcg6c99fc,5852
 fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=YGgzTspkK9ItSMzwuYv_1tY7_1g89Qpeny5Auinxk1E,2708
@@ -88,7 +88,7 @@ fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_T
 fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=n9DAn2Rn_4gUphJCJk4CaQH7WQP4nBNZxqF9dj0H5cw,5768
 fractal_server/app/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=O1bNg1DiSDJmQE0RmOk2Ii47DagiXp5ryd0R6KxO2OM,3177
-fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=1NoXMQH-JAzBHGmzlUdu6P1gLGJE5y17U3yLxJOyyHE,5045
+fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=6E1Pl2k9DFekZkZgU3rpp_v8wrJG7qGt4rDaUCFlnXI,5059
 fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
 fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
 fractal_server/app/runner/run_subprocess.py,sha256=c3JbYXq3hX2aaflQU19qJ5Xs6J6oXGNvnTEoAfv2bxc,959
@@ -99,11 +99,11 @@ fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 fractal_server/app/runner/v2/_local.py,sha256=DK8yagbvd6HHjcDVhUzTy0f7MURlTkQha-NM6OZKgJc,3044
 fractal_server/app/runner/v2/_slurm_ssh.py,sha256=_bytOf8z9sdrhI03D6eqg-aQPnJ7V2-qnqpcHAYizns,3278
 fractal_server/app/runner/v2/_slurm_sudo.py,sha256=DBCNxifXmMkpu71Wnk5u9-wKT7PV1WROQuY_4DYoZRI,2993
-fractal_server/app/runner/v2/db_tools.py,sha256=jj5Jlm5xPXIxpxbTyVP9QJ_8wxkzdWFU9DzgLW-8fhs,2420
+fractal_server/app/runner/v2/db_tools.py,sha256=tyLGY-g4ISZSJzk6ootEuHocdVwMg3taBL0oIivXV7M,2846
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
 fractal_server/app/runner/v2/runner.py,sha256=SsKEZAsB8sPV8W3khTkAqaGdDwoTm_lav-fx6DdCwyA,15294
-fractal_server/app/runner/v2/runner_functions.py,sha256=GRx1oZzW1KCWq8WS57OY2XFI8cDqBk6Bv-CEeWqIaHQ,16467
+fractal_server/app/runner/v2/runner_functions.py,sha256=5cK5O2rTrCsCxMTVN3iNPRwZ_891BC9_RMo64a8ZGYw,16338
 fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=9t1CHN3EyfsGRWfG257YPY5WjQ6zuztsw_KZrpEAFPo,3703
 fractal_server/app/runner/v2/submit_workflow.py,sha256=EDUyUuIPwZHb2zm7SCRRoFsGq2cN-b5OKw6CYkZ8kWk,13048
 fractal_server/app/runner/v2/task_interface.py,sha256=IXdQTI8rXFgXv1Ez0js4CjKFf3QwO2GCHRTuwiFtiTQ,2891
@@ -208,8 +208,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENP
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a15.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a15.dist-info/METADATA,sha256=HRQC8Dsp32xLGZ_ThsguhcO2XvKgLuHwo0ZDUEGBLu8,4563
-fractal_server-2.14.0a15.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.0a15.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a15.dist-info/RECORD,,
+fractal_server-2.14.0a17.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a17.dist-info/METADATA,sha256=Pv9LhU977x4Xvcet2GgaVMTnlU1ZQyKARhRTwH6_GkQ,4563
+fractal_server-2.14.0a17.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.0a17.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a17.dist-info/RECORD,,