fractal-server 2.14.0a21__py3-none-any.whl → 2.14.0a23__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a21"
+__VERSION__ = "2.14.0a23"

fractal_server/app/routes/api/v2/history.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Any
 from typing import Optional
 
@@ -92,7 +93,7 @@ async def get_workflow_tasks_statuses(
         db=db,
     )
 
-    response = {}
+    response: dict[int, dict[str, int | str] | None] = {}
     for wftask in workflow.task_list:
         res = await db.execute(
             select(HistoryRun)
@@ -130,7 +131,18 @@ async def get_workflow_tasks_statuses(
                 f"num_{target_status.value}_images"
             ] = num_images
 
-    return JSONResponse(content=response, status_code=200)
+    new_response = deepcopy(response)
+    for key, value in response.items():
+        if value is not None:
+            num_total_images = sum(
+                value[f"num_{target_status.value}_images"]
+                for target_status in HistoryUnitStatus
+            )
+            if num_total_images > value["num_available_images"]:
+                value["num_available_images"] = None
+        new_response[key] = value
+
+    return JSONResponse(content=new_response, status_code=200)
 
 
 @router.get("/project/{project_id}/status/run/")

fractal_server/app/runner/compress_folder.py

@@ -11,8 +11,8 @@ built-in `tarfile` library has to do with performance issues we observed
 when handling files which were just created within a SLURM job, and in the
 context of a CephFS filesystem.
 """
-import shutil
 import sys
+import time
 from pathlib import Path
 
 from fractal_server.app.runner.run_subprocess import run_subprocess
@@ -20,48 +20,66 @@ from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
 
 
-def copy_subfolder(src: Path, dest: Path, logger_name: str):
+def _copy_subfolder(src: Path, dest: Path, logger_name: str):
+    t_start = time.perf_counter()
     cmd_cp = f"cp -r {src.as_posix()} {dest.as_posix()}"
     logger = get_logger(logger_name=logger_name)
     logger.debug(f"{cmd_cp=}")
     res = run_subprocess(cmd=cmd_cp, logger_name=logger_name)
+    elapsed = time.perf_counter() - t_start
+    logger.debug(f"[_copy_subfolder] END {elapsed=} s ({dest.as_posix()})")
     return res
 
 
-def create_tar_archive(
-    tarfile_path: Path,
+def _create_tar_archive(
+    tarfile_path: str,
     subfolder_path_tmp_copy: Path,
     logger_name: str,
-    remote_to_local: bool,
+    filelist_path: str | None,
 ):
     logger = get_logger(logger_name)
-
-    if remote_to_local:
-        exclude_options = "--exclude *sbatch --exclude *_in_*.pickle "
+    logger.debug(f"[_create_tar_archive] START ({tarfile_path})")
+    t_start = time.perf_counter()
+
+    if filelist_path is None:
+        cmd_tar = (
+            f"tar -c -z -f {tarfile_path} "
+            f"--directory={subfolder_path_tmp_copy.as_posix()} "
+            "."
+        )
     else:
-        exclude_options = ""
+        cmd_tar = (
+            f"tar -c -z -f {tarfile_path} "
+            f"--directory={subfolder_path_tmp_copy.as_posix()} "
+            f"--files-from={filelist_path} --ignore-failed-read"
+        )
+
+    logger.critical(f"cmd tar:\n{cmd_tar}")
 
-    cmd_tar = (
-        f"tar czf {tarfile_path} "
-        f"{exclude_options} "
-        f"--directory={subfolder_path_tmp_copy.as_posix()} "
-        "."
-    )
-    logger.debug(f"cmd tar:\n{cmd_tar}")
     run_subprocess(cmd=cmd_tar, logger_name=logger_name, allow_char="*")
+    elapsed = time.perf_counter() - t_start
+    logger.debug(f"[_create_tar_archive] END {elapsed=} s ({tarfile_path})")
 
 
-def remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
+def _remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
     logger = get_logger(logger_name)
+    t_start = time.perf_counter()
     try:
-        logger.debug(f"Now remove {subfolder_path_tmp_copy}")
-        shutil.rmtree(subfolder_path_tmp_copy)
+        cmd_rm = f"rm -rf {subfolder_path_tmp_copy}"
+        logger.debug(f"cmd rm:\n{cmd_rm}")
+        run_subprocess(cmd=cmd_rm, logger_name=logger_name, allow_char="*")
     except Exception as e:
-        logger.debug(f"ERROR during shutil.rmtree: {e}")
+        logger.debug(f"ERROR during {cmd_rm}: {e}")
+    elapsed = time.perf_counter() - t_start
+    logger.debug(
+        f"[_remove_temp_subfolder] END {elapsed=} s "
+        f"({subfolder_path_tmp_copy=})"
+    )
 
 
 def compress_folder(
-    subfolder_path: Path, remote_to_local: bool = False
+    subfolder_path: Path,
+    filelist_path: str | None,
 ) -> str:
     """
     Compress e.g. `/path/archive` into `/path/archive.tar.gz`
@@ -91,14 +109,16 @@ def compress_folder(
         subfolder_path.parent / f"{subfolder_path.name}_copy"
     )
     try:
-        copy_subfolder(
-            subfolder_path, subfolder_path_tmp_copy, logger_name=logger_name
+        _copy_subfolder(
+            subfolder_path,
+            subfolder_path_tmp_copy,
+            logger_name=logger_name,
         )
-        create_tar_archive(
+        _create_tar_archive(
             tarfile_path,
             subfolder_path_tmp_copy,
             logger_name=logger_name,
-            remote_to_local=remote_to_local,
+            filelist_path=filelist_path,
         )
         return tarfile_path
 
@@ -107,7 +127,9 @@ def compress_folder(
         sys.exit(1)
 
     finally:
-        remove_temp_subfolder(subfolder_path_tmp_copy, logger_name=logger_name)
+        _remove_temp_subfolder(
+            subfolder_path_tmp_copy, logger_name=logger_name
+        )
 
 
 def main(sys_argv: list[str]):
@@ -115,15 +137,21 @@ def main(sys_argv: list[str]):
     help_msg = (
         "Expected use:\n"
         "python -m fractal_server.app.runner.compress_folder "
-        "path/to/folder [--remote-to-local]\n"
+        "path/to/folder [--filelist /path/to/filelist]\n"
     )
     num_args = len(sys_argv[1:])
     if num_args == 0:
         sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
     elif num_args == 1:
-        compress_folder(subfolder_path=Path(sys_argv[1]))
-    elif num_args == 2 and sys_argv[2] == "--remote-to-local":
-        compress_folder(subfolder_path=Path(sys_argv[1]), remote_to_local=True)
+        compress_folder(
+            subfolder_path=Path(sys_argv[1]),
+            filelist_path=None,
+        )
+    elif num_args == 3 and sys_argv[2] == "--filelist":
+        compress_folder(
+            subfolder_path=Path(sys_argv[1]),
+            filelist_path=sys_argv[3],
+        )
     else:
         sys.exit(f"Invalid argument.\n{help_msg}\nProvided: {sys_argv[1:]=}")
 

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -60,6 +60,7 @@ class BaseSlurmRunner(BaseRunner):
         root_dir_local: Path,
         root_dir_remote: Path,
         slurm_runner_type: Literal["ssh", "sudo"],
+        python_worker_interpreter: str,
         common_script_lines: Optional[list[str]] = None,
         user_cache_dir: Optional[str] = None,
         poll_interval: Optional[int] = None,
@@ -70,6 +71,7 @@ class BaseSlurmRunner(BaseRunner):
         self.common_script_lines = common_script_lines or []
         self._check_slurm_account()
         self.user_cache_dir = user_cache_dir
+        self.python_worker_interpreter = python_worker_interpreter
 
         settings = Inject(get_settings)
 
@@ -327,9 +329,9 @@ class BaseSlurmRunner(BaseRunner):
         )
         logger.info("[_submit_single_sbatch] END")
 
-    def _copy_files_from_remote_to_local(
+    def _fetch_artifacts(
         self,
-        slurm_job: SlurmJob,
+        finished_slurm_jobs: list[SlurmJob],
     ) -> None:
         raise NotImplementedError("Implement in child class.")
 
@@ -530,14 +532,14 @@ class BaseSlurmRunner(BaseRunner):
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[submit] {finished_job_ids=}")
-
+            finished_jobs = [
+                self.jobs[_slurm_job_id] for _slurm_job_id in finished_job_ids
+            ]
+            self._fetch_artifacts(finished_jobs)
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
                     logger.debug(f"[submit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
-                    self._copy_files_from_remote_to_local(
-                        slurm_job
-                    )  # FIXME: add prefix  # noqa
                     was_job_scancelled = slurm_job_id in scancelled_job_ids
                     result, exception = self._postprocess_single_task(
                         task=slurm_job.tasks[0],
@@ -653,7 +655,9 @@ class BaseSlurmRunner(BaseRunner):
         if len(args_batches) != math.ceil(tot_tasks / tasks_per_job):
             raise RuntimeError("Something wrong here while batching tasks")
 
-        logger.info(f"START submission phase, {list(self.jobs.keys())=}")
+        # Part 1/3: Iterate over chunks, prepare SlurmJob objects
+        logger.info("[multisubmit] Prepare `SlurmJob`s.")
+        jobs_to_submit = []
         for ind_batch, chunk in enumerate(args_batches):
             prefix = f"{MULTISUBMIT_PREFIX}-{ind_batch:06d}"
             tasks = []
@@ -673,17 +677,26 @@ class BaseSlurmRunner(BaseRunner):
                     ),
                 )
 
-            slurm_job = SlurmJob(
-                prefix=prefix,
-                workdir_local=workdir_local,
-                workdir_remote=workdir_remote,
-                tasks=tasks,
+            jobs_to_submit.append(
+                SlurmJob(
+                    prefix=prefix,
+                    workdir_local=workdir_local,
+                    workdir_remote=workdir_remote,
+                    tasks=tasks,
+                )
             )
+
+        # FIXME: split parts 2 and 3
+        # Part 2/3. Transfer all relevant input files (for SSH)
+        # Part 3/3. Run all `sbatch`es and update `self.jobs`
+        logger.info("[multisubmit] Transfer files and submit jobs.")
+        for slurm_job in jobs_to_submit:
             self._submit_single_sbatch(
                 func,
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
+
         if task_type == "parallel":
             # FIXME: replace loop with a `bulk_update_history_unit` function
             for ind, task_files in enumerate(list_task_files):
@@ -711,20 +724,21 @@ class BaseSlurmRunner(BaseRunner):
 
         # Retrieval phase
         logger.info("[multisubmit] START retrieval phase")
+        scancelled_job_ids = []
         while len(self.jobs) > 0:
 
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[multisubmit] {finished_job_ids=}")
+            finished_jobs = [
+                self.jobs[_slurm_job_id] for _slurm_job_id in finished_job_ids
+            ]
+            self._fetch_artifacts(finished_jobs)
 
-            scancelled_job_ids = []
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
                     logger.info(f"[multisubmit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
-                    self._copy_files_from_remote_to_local(
-                        slurm_job
-                    )  # FIXME: add prefix  # noqa
                     for task in slurm_job.tasks:
                         logger.info(f"[multisubmit] Now process {task.index=}")
                         was_job_scancelled = slurm_job_id in scancelled_job_ids
@@ -810,3 +824,19 @@ class BaseSlurmRunner(BaseRunner):
                 )
         logger.info("[scancel_jobs] END")
         return scancelled_job_ids
+
+    def validate_slurm_jobs_workdirs(
+        self,
+        slurm_jobs: list[SlurmJob],
+    ) -> None:
+        """
+        Check that a list of `SlurmJob`s have homogeneous working folders.
+        """
+        # Extract `workdir_remote` and `workdir_local`
+        set_workdir_local = set(_job.workdir_local for _job in slurm_jobs)
+        set_workdir_remote = set(_job.workdir_remote for _job in slurm_jobs)
+
+        if len(set_workdir_local) > 1:
+            raise ValueError(f"Non-unique values in {set_workdir_local=}.")
+        if len(set_workdir_remote) > 1:
+            raise ValueError(f"Non-unique values in {set_workdir_remote=}.")

fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py

@@ -20,31 +20,47 @@ class SlurmTask(BaseModel):
     index: int
 
     @property
-    def input_pickle_file_local(self) -> str:
+    def input_pickle_file_local_path(self) -> Path:
         return (
             self.workdir_local / f"{self.prefix}-{self.component}-input.pickle"
-        ).as_posix()
+        )
 
     @property
-    def input_pickle_file_remote(self) -> str:
+    def input_pickle_file_remote_path(self) -> Path:
         return (
             self.workdir_remote
             / f"{self.prefix}-{self.component}-input.pickle"
-        ).as_posix()
+        )
 
     @property
-    def output_pickle_file_local(self) -> str:
+    def output_pickle_file_local_path(self) -> Path:
         return (
             self.workdir_local
             / f"{self.prefix}-{self.component}-output.pickle"
-        ).as_posix()
+        )
 
     @property
-    def output_pickle_file_remote(self) -> str:
+    def output_pickle_file_remote_path(self) -> Path:
         return (
             self.workdir_remote
             / f"{self.prefix}-{self.component}-output.pickle"
-        ).as_posix()
+        )
+
+    @property
+    def input_pickle_file_local(self) -> str:
+        return self.input_pickle_file_local_path.as_posix()
+
+    @property
+    def input_pickle_file_remote(self) -> str:
+        return self.input_pickle_file_remote_path.as_posix()
+
+    @property
+    def output_pickle_file_local(self) -> str:
+        return self.output_pickle_file_local_path.as_posix()
+
+    @property
+    def output_pickle_file_remote(self) -> str:
+        return self.output_pickle_file_remote_path.as_posix()
 
 
 class SlurmJob(BaseModel):
@@ -74,29 +90,45 @@ class SlurmJob(BaseModel):
             return "%j"
 
     @property
-    def slurm_stdout_remote(self) -> str:
+    def slurm_stdout_remote_path(self) -> Path:
         return (
             self.workdir_remote
             / f"{self.prefix}-slurm-{self.slurm_job_id_placeholder}.out"
-        ).as_posix()
+        )
 
     @property
-    def slurm_stderr_remote(self) -> str:
+    def slurm_stdout_remote(self) -> str:
+        return self.slurm_stdout_remote_path.as_posix()
+
+    @property
+    def slurm_stderr_remote_path(self) -> Path:
         return (
             self.workdir_remote
             / f"{self.prefix}-slurm-{self.slurm_job_id_placeholder}.err"
-        ).as_posix()
+        )
 
     @property
-    def slurm_stdout_local(self) -> str:
+    def slurm_stderr_remote(self) -> str:
+        return self.slurm_stderr_remote_path.as_posix()
+
+    @property
+    def slurm_stdout_local_path(self) -> str:
         return (
             self.workdir_local
             / f"{self.prefix}-slurm-{self.slurm_job_id_placeholder}.out"
-        ).as_posix()
+        )
 
     @property
-    def slurm_stderr_local(self) -> str:
+    def slurm_stdout_local(self) -> str:
+        return self.slurm_stdout_local_path.as_posix()
+
+    @property
+    def slurm_stderr_local_path(self) -> Path:
         return (
             self.workdir_local
             / f"{self.prefix}-slurm-{self.slurm_job_id_placeholder}.err"
-        ).as_posix()
+        )
+
+    @property
+    def slurm_stderr_local(self) -> str:
+        return self.slurm_stderr_local_path.as_posix()

fractal_server/app/runner/executors/slurm_ssh/runner.py

@@ -38,7 +38,6 @@ class SlurmSSHRunner(BaseSlurmRunner):
         logger.warning(self.fractal_ssh)
 
         settings = Inject(get_settings)
-        self.python_worker_interpreter = settings.FRACTAL_SLURM_WORKER_PYTHON
 
         super().__init__(
             slurm_runner_type="ssh",
@@ -47,6 +46,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
             common_script_lines=common_script_lines,
             user_cache_dir=user_cache_dir,
             poll_interval=poll_interval,
+            python_worker_interpreter=settings.FRACTAL_SLURM_WORKER_PYTHON,
         )
 
     def _mkdir_local_folder(self, folder: str) -> None:
@@ -58,86 +58,81 @@ class SlurmSSHRunner(BaseSlurmRunner):
             parents=True,
         )
 
-    def _copy_files_from_remote_to_local(self, slurm_job: SlurmJob) -> None:
-        self._get_subfolder_sftp(job=slurm_job)
-
-    def _put_subfolder_sftp(self, job: SlurmJob) -> None:
-        # FIXME re-introduce use of this function, but only after splitting
-        # submission logic into
-        # 1. prepare all
-        # 2. send folder
-        # 3. submit all
+    def _fetch_artifacts(
+        self,
+        finished_slurm_jobs: list[SlurmJob],
+    ) -> None:
         """
-        Transfer the jobs subfolder to the remote host.
+        Fetch artifacts for a list of SLURM jobs.
         """
 
-        # Create local archive
-        tarfile_path_local = compress_folder(job.workdir_local)
-        tarfile_name = Path(tarfile_path_local).name
-        logger.info(f"Subfolder archive created at {tarfile_path_local}")
+        # Check length
+        if len(finished_slurm_jobs) == 0:
+            logger.debug(f"[_fetch_artifacts] EXIT ({finished_slurm_jobs=}).")
+            return None
 
-        # Transfer archive
-        tarfile_path_remote = (
-            job.workdir_remote.parent / tarfile_name
-        ).as_posix()
-        t_0_put = time.perf_counter()
-        self.fractal_ssh.send_file(
-            local=tarfile_path_local,
-            remote=tarfile_path_remote,
-        )
-        t_1_put = time.perf_counter()
-        logger.info(
-            f"Subfolder archive transferred to {tarfile_path_remote}"
-            f" - elapsed: {t_1_put - t_0_put:.3f} s"
+        t_0 = time.perf_counter()
+        logger.debug(
+            f"[_fetch_artifacts] START ({len(finished_slurm_jobs)=})."
         )
 
-        # Remove local archive
-        Path(tarfile_path_local).unlink()
-        logger.debug(f"Local archive {tarfile_path_local} removed")
-
-        # Uncompress remote archive
-        tar_command = (
-            f"{self.python_worker_interpreter} -m "
-            "fractal_server.app.runner.extract_archive "
-            f"{tarfile_path_remote}"
-        )
-        self.fractal_ssh.run_command(cmd=tar_command)
+        # Extract `workdir_remote` and `workdir_local`
+        self.validate_slurm_jobs_workdirs(finished_slurm_jobs)
+        workdir_local = finished_slurm_jobs[0].workdir_local
+        workdir_remote = finished_slurm_jobs[0].workdir_remote
 
-    def _get_subfolder_sftp(self, job: SlurmJob) -> None:
-        """
-        Fetch a remote folder via tar+sftp+tar
-        """
-
-        t_0 = time.perf_counter()
-        logger.debug("[_get_subfolder_sftp] Start")
+        # Define local/remote tarfile paths
         tarfile_path_local = (
-            job.workdir_local.parent / f"{job.workdir_local.name}.tar.gz"
+            workdir_local.parent / f"{workdir_local.name}.tar.gz"
         ).as_posix()
         tarfile_path_remote = (
-            job.workdir_remote.parent / f"{job.workdir_remote.name}.tar.gz"
+            workdir_remote.parent / f"{workdir_remote.name}.tar.gz"
         ).as_posix()
 
-        # Remove remote tarfile
-        try:
-            rm_command = f"rm {tarfile_path_remote}"
-            self.fractal_ssh.run_command(cmd=rm_command)
-            logger.info(f"Removed {tarfile_path_remote=}")
-        except RuntimeError as e:
-            logger.info(
-                f"Could not remove {tarfile_path_remote=}.\n"
-                f"Original error: {str(e)}"
-            )
+        # Create file list
+        # # FIXME can we make this more efficient with iterations?
+        filelist = []
+        for _slurm_job in finished_slurm_jobs:
+            _single_job_filelist = [
+                _slurm_job.slurm_stdout_remote_path.name,
+                _slurm_job.slurm_stderr_remote_path.name,
+            ]
+            for task in _slurm_job.tasks:
+                _single_job_filelist.extend(
+                    [
+                        task.output_pickle_file_remote_path.name,
+                        task.task_files.log_file_remote_path.name,
+                        task.task_files.args_file_remote_path.name,
+                        task.task_files.metadiff_file_remote_path.name,
+                    ]
+                )
+            filelist.extend(_single_job_filelist)
+        filelist_string = "\n".join(filelist)
+        elapsed = time.perf_counter() - t_0
+        logger.debug(
+            "[_fetch_artifacts] Created filelist "
+            f"({len(filelist)=}, from start: {elapsed:.3f} s)."
+        )
+
+        # Write filelist to file remotely
+        tmp_filelist_path = workdir_remote / f"filelist_{time.time()}.txt"
+        self.fractal_ssh.write_remote_file(
+            path=tmp_filelist_path.as_posix(),
+            content=f"{filelist_string}\n",
+        )
+        elapsed = time.perf_counter() - t_0
+        logger.debug(
+            f"[_fetch_artifacts] File list written to {tmp_filelist_path} "
+            f"(from start: {elapsed:.3f} s)."
+        )
 
         # Create remote tarfile
-        # FIXME: introduce filtering by prefix, so that when the subfolder
-        # includes N SLURM jobs we don't always copy the cumulative folder
-        # but only the relevant part
         t_0_tar = time.perf_counter()
         tar_command = (
             f"{self.python_worker_interpreter} "
             "-m fractal_server.app.runner.compress_folder "
-            f"{job.workdir_remote.as_posix()} "
-            "--remote-to-local"
+            f"{workdir_remote.as_posix()} "
+            f"--filelist {tmp_filelist_path}"
         )
         self.fractal_ssh.run_command(cmd=tar_command)
         t_1_tar = time.perf_counter()
@@ -167,6 +162,47 @@ class SlurmSSHRunner(BaseSlurmRunner):
         t_1 = time.perf_counter()
         logger.info(f"[_get_subfolder_sftp] End - elapsed: {t_1 - t_0:.3f} s")
 
+    def _send_inputs(self, jobs: list[SlurmJob]) -> None:
+        """
+        Transfer the jobs subfolder to the remote host.
+        """
+        for job in jobs:
+
+            # Create local archive
+            tarfile_path_local = compress_folder(
+                job.workdir_local,
+                filelist_path=None,
+            )
+            tarfile_name = Path(tarfile_path_local).name
+            logger.info(f"Subfolder archive created at {tarfile_path_local}")
+
+            # Transfer archive
+            tarfile_path_remote = (
+                job.workdir_remote.parent / tarfile_name
+            ).as_posix()
+            t_0_put = time.perf_counter()
+            self.fractal_ssh.send_file(
+                local=tarfile_path_local,
+                remote=tarfile_path_remote,
+            )
+            t_1_put = time.perf_counter()
+            logger.info(
+                f"Subfolder archive transferred to {tarfile_path_remote}"
+                f" - elapsed: {t_1_put - t_0_put:.3f} s"
+            )
+
+            # Remove local archive
+            Path(tarfile_path_local).unlink()
+            logger.debug(f"Local archive {tarfile_path_local} removed")
+
+            # Uncompress remote archive
+            tar_command = (
+                f"{self.python_worker_interpreter} -m "
+                "fractal_server.app.runner.extract_archive "
+                f"{tarfile_path_remote}"
+            )
+            self.fractal_ssh.run_command(cmd=tar_command)
+
     def _run_remote_cmd(self, cmd: str) -> str:
         stdout = self.fractal_ssh.run_command(cmd=cmd)
         return stdout

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -3,6 +3,7 @@ import os
 import shlex
 import subprocess  # nosec
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Optional
 
@@ -15,7 +16,6 @@ from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
-
 logger = set_logger(__name__)
 
 
@@ -67,10 +67,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
         self.slurm_account = slurm_account
         settings = Inject(get_settings)
 
-        self.python_worker_interpreter = (
-            settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
-        )
-
         super().__init__(
             slurm_runner_type="sudo",
             root_dir_local=root_dir_local,
@@ -78,6 +74,9 @@ class SudoSlurmRunner(BaseSlurmRunner):
             common_script_lines=common_script_lines,
             user_cache_dir=user_cache_dir,
             poll_interval=poll_interval,
+            python_worker_interpreter=(
+                settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
+            ),
         )
 
     def _mkdir_local_folder(self, folder: str) -> None:
@@ -88,12 +87,12 @@ class SudoSlurmRunner(BaseSlurmRunner):
     def _mkdir_remote_folder(self, folder: str) -> None:
         _mkdir_as_user(folder=folder, user=self.slurm_user)
 
-    def _copy_files_from_remote_to_local(self, job: SlurmJob) -> None:
+    def _fetch_artifacts_single_job(self, job: SlurmJob) -> None:
         """
-        Note: this would differ for SSH
+        Fetch artifacts for a single SLURM jobs.
         """
         logger.debug(
-            f"[_copy_files_from_remote_to_local] {job.slurm_job_id=} START"
+            f"[_fetch_artifacts_single_job] {job.slurm_job_id=} START"
         )
         source_target_list = [
             (job.slurm_stdout_remote, job.slurm_stdout_local),
@@ -140,9 +139,30 @@ class SudoSlurmRunner(BaseSlurmRunner):
                     f"SKIP copy {source} into {target}. "
                     f"Original error: {str(e)}"
                 )
+        logger.debug(f"[_fetch_artifacts_single_job] {job.slurm_job_id=} END")
+
+    def _fetch_artifacts(
+        self,
+        finished_slurm_jobs: list[SlurmJob],
+    ) -> None:
+        """
+        Fetch artifacts for a list of SLURM jobs.
+        """
+        MAX_NUM_THREADS = 4
+        THREAD_NAME_PREFIX = "fetch_artifacts"
         logger.debug(
-            f"[_copy_files_from_remote_to_local] {job.slurm_job_id=} END"
+            "[_fetch_artifacts] START "
+            f"({MAX_NUM_THREADS=}, {len(finished_slurm_jobs)=})."
         )
+        with ThreadPoolExecutor(
+            max_workers=MAX_NUM_THREADS,
+            thread_name_prefix=THREAD_NAME_PREFIX,
+        ) as executor:
+            executor.map(
+                self._fetch_artifacts_single_job,
+                finished_slurm_jobs,
+            )
+        logger.debug("[_fetch_artifacts] END.")
 
     def _run_remote_cmd(self, cmd: str) -> str:
         res = _run_command_as_user(

fractal_server/app/runner/extract_archive.py

@@ -57,9 +57,7 @@ def extract_archive(archive_path: Path):
 
     # Run tar command
     cmd_tar = (
-        f"tar -xzvf {archive_path} "
-        f"--directory={subfolder_path.as_posix()} "
-        "."
+        f"tar -xzvf {archive_path} " f"--directory={subfolder_path.as_posix()}"
     )
     logger.debug(f"{cmd_tar=}")
     run_subprocess(cmd=cmd_tar, logger_name=logger_name)

fractal_server/app/runner/task_files.py

@@ -85,11 +85,15 @@ class TaskFiles(BaseModel):
         ).as_posix()
 
     @property
-    def log_file_remote(self) -> str:
+    def log_file_remote_path(self) -> Path:
         self._check_component()
         return (
             self.wftask_subfolder_remote / f"{self.prefix_component}-log.txt"
-        ).as_posix()
+        )
+
+    @property
+    def log_file_remote(self) -> str:
+        return self.log_file_remote_path.as_posix()
 
     @property
     def args_file_local(self) -> str:
@@ -99,11 +103,15 @@ class TaskFiles(BaseModel):
         ).as_posix()
 
     @property
-    def args_file_remote(self) -> str:
+    def args_file_remote_path(self) -> Path:
         self._check_component()
         return (
             self.wftask_subfolder_remote / f"{self.prefix_component}-args.json"
-        ).as_posix()
+        )
+
+    @property
+    def args_file_remote(self) -> str:
+        return self.args_file_remote_path.as_posix()
 
     @property
     def metadiff_file_local(self) -> str:
@@ -114,12 +122,16 @@ class TaskFiles(BaseModel):
         ).as_posix()
 
     @property
-    def metadiff_file_remote(self) -> str:
+    def metadiff_file_remote_path(self) -> Path:
         self._check_component()
         return (
             self.wftask_subfolder_remote
             / f"{self.prefix_component}-metadiff.json"
-        ).as_posix()
+        )
+
+    @property
+    def metadiff_file_remote(self) -> str:
+        return self.metadiff_file_remote_path.as_posix()
 
     @property
     def remote_files_dict(self) -> dict[str, str]:

fractal_server/ssh/_fabric.py

@@ -501,7 +501,8 @@ class FractalSSH(object):
             content: Contents to be written to file.
             lock_timeout: Timeout for lock acquisition (overrides default).
         """
-        self.logger.info(f"START writing to remote file {path}.")
+        t_start = time.perf_counter()
+        self.logger.info(f"[write_remote_file] START ({path}).")
         actual_lock_timeout = self.default_lock_timeout
         if lock_timeout is not None:
             actual_lock_timeout = lock_timeout
@@ -518,7 +519,8 @@ class FractalSSH(object):
                     e=e, message=f"Error in `write_remote_file`, for {path=}."
                 )
 
-        self.logger.info(f"END writing to remote file {path}.")
+        elapsed = time.perf_counter() - t_start
+        self.logger.info(f"[write_remote_file] END, {elapsed=} s ({path}).")
 
     def remote_exists(self, path: str) -> bool:
         """

{fractal_server-2.14.0a21.dist-info → fractal_server-2.14.0a23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a21
+Version: 2.14.0a23
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.0a21.dist-info → fractal_server-2.14.0a23.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=X5Dy_f87GBiFeLzs2riLgudM2HP43U0ZuXNsU2NF7Os,26
+fractal_server/__init__.py,sha256=HmMn7o6LFtf7DorxF19ycJBUGPrTwAzfEUuVJ_cF14Q,26
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,7 +36,7 @@ fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=ZlI6nwzB5r9AiY
 fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=qdXCb6IP8-qPEAxGZKljtjIqNzIAyRaAsQSRi5VqFHM,6773
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=uhNSs-jcS7ndIUFKiOC1yrDiViw3uvKEXi9UL04BMks,11642
 fractal_server/app/routes/api/v2/dataset.py,sha256=h5AhE0sdhQ20ZlIbEJsFnHIOUW0S1VHFpoflpBkVScs,8936
-fractal_server/app/routes/api/v2/history.py,sha256=lMbaybooBzzbCgD9vdzPyNxdgAZuzCH_YrW9ost-UgI,17253
+fractal_server/app/routes/api/v2/history.py,sha256=FvZGl66hIdo70GvWoOhRQ__knbkbp5u440sl6qhj7nA,17748
 fractal_server/app/routes/api/v2/images.py,sha256=BGpO94gVd8BTpCN6Mun2RXmjrPmfkIp73m8RN7uiGW4,8361
 fractal_server/app/routes/api/v2/job.py,sha256=MU1sHIKk_89WrD0TD44d4ufzqnywot7On_W71KjyUbQ,6500
 fractal_server/app/routes/api/v2/project.py,sha256=uAZgATiHcOvbnRX-vv1D3HoaEUvLUd7vzVmGcqOP8ZY,4602
@@ -67,7 +67,7 @@ fractal_server/app/routes/aux/validate_user_settings.py,sha256=FLVi__8YFcm_6c_K5
 fractal_server/app/routes/pagination.py,sha256=L8F5JqekF39qz-LpeScdlhb57MQnSRXjK4ZEtsZqYLk,1210
 fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
-fractal_server/app/runner/compress_folder.py,sha256=HSc1tv7x2DBjBoXwugZlC79rm9GNBIWtQKK9yWn5ZBI,3991
+fractal_server/app/runner/compress_folder.py,sha256=LswrzzW7h8LJzwNcVIdhhTwHMYo2-dua3BeO272DCaw,4879
 fractal_server/app/runner/exceptions.py,sha256=JC5ufHyeA1hYD_rkZUscI30DD8D903ncag7Z3AArmUY,4215
 fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/base_runner.py,sha256=_elfqkuw1TGRacPG4aTTYKUTbF4A-Y2Rmft9LFi-Mwc,5554
@@ -78,22 +78,22 @@ fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=fZaFUUXqDH0p3DndCFUpFqTqyD2tMVCuSYgYLAycpVw,15897
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=f01IDAtBoatOYEP4UtrH0Y4qN7BwM1ov4Bx8rotQg1M,31099
+fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=UIQ3W0HGXLw8vGt-WeqFq_UafXtyPbuyDAAyXbV96Vo,32314
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=-fAX1DZMB5RZnyYanIJD72mWOJAPkh21jd4loDXKJw4,5994
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=FS_F8EaPp-A5eQT5_ZH3ICCHt0-C8b_2OSYcyRkXnb4,5851
-fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=YGgzTspkK9ItSMzwuYv_1tY7_1g89Qpeny5Auinxk1E,2708
+fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=RoxHLKOn0_wGjnY0Sv0a9nDSiqxYZHKRoMkT3p9_G1E,3607
 fractal_server/app/runner/executors/slurm_common/utils_executors.py,sha256=naPyJI0I3lD-sYHbSXbMFGUBK4h_SggA5V91Z1Ch1Xg,1416
 fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=WhcrcvDMvSoQ-GroDXWzIfESbOLxMf3m-ZE16pbqQRg,5777
+fractal_server/app/runner/executors/slurm_ssh/runner.py,sha256=B_L4MczNPoOIqJPHYmJwwVRl5nLsVRecIcyW-OrgU6E,7138
 fractal_server/app/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=O1bNg1DiSDJmQE0RmOk2Ii47DagiXp5ryd0R6KxO2OM,3177
-fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=FdhENPwz_efXSPqG3saEvJOUiv6uG1vd0ET61QmbC4o,5136
-fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
+fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=WGGVHX_juqyC6OVhln9yg-YKjLiuAoWZhAGxBjhNkWw,5873
+fractal_server/app/runner/extract_archive.py,sha256=iQOsJNYW-ae1TCK5WqmBvmqw7FPEaEh3U7UivVCLayY,2462
 fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
 fractal_server/app/runner/run_subprocess.py,sha256=c3JbYXq3hX2aaflQU19qJ5Xs6J6oXGNvnTEoAfv2bxc,959
 fractal_server/app/runner/set_start_and_last_task_index.py,sha256=-q4zVybAj8ek2XlbENKlfOAJ39hT_zoJoZkqzDqiAMY,1254
 fractal_server/app/runner/shutdown.py,sha256=9pfSKHDNdIcm0eY-opgRTi7y0HmvfPmYiu9JR6Idark,2082
-fractal_server/app/runner/task_files.py,sha256=KfjDqQV9b9wImusxJtQfVST42FyvT3Kza0WooI-6OBk,3198
+fractal_server/app/runner/task_files.py,sha256=wWp7GjZAt04X-N5M1YHDRyWHeNz8kFjK-jSA6DaxC20,3510
 fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/v2/_local.py,sha256=DK8yagbvd6HHjcDVhUzTy0f7MURlTkQha-NM6OZKgJc,3044
 fractal_server/app/runner/v2/_slurm_ssh.py,sha256=_bytOf8z9sdrhI03D6eqg-aQPnJ7V2-qnqpcHAYizns,3278
@@ -177,7 +177,7 @@ fractal_server/migrations/versions/f384e1c0cf5d_drop_task_default_args_columns.p
 fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py,sha256=TDWCaIoM0Q4SpRWmR9zr_rdp3lJXhCfBPTMhtrP5xYE,3950
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
-fractal_server/ssh/_fabric.py,sha256=lNy4IX1I4We6VoWa4Bz4fUPuApLMSoejpyE6I3jDZeM,22869
+fractal_server/ssh/_fabric.py,sha256=gnSv_DaQ8uYLS35Rqb84wo3HRkMazXGVd-D19fo9zqA,22967
 fractal_server/string_tools.py,sha256=niViRrrZAOo0y6pEFI9L_eUYS1PoOiQZUBtngiLc2_k,1877
 fractal_server/syringe.py,sha256=3qSMW3YaMKKnLdgnooAINOPxnCOxP7y2jeAQYB21Gdo,2786
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
@@ -207,8 +207,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENP
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a21.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a21.dist-info/METADATA,sha256=OZesW99axIUpT41CcJOkIszG_F5jLrDfKLT29EctIgI,4563
-fractal_server-2.14.0a21.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.0a21.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a21.dist-info/RECORD,,
+fractal_server-2.14.0a23.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a23.dist-info/METADATA,sha256=qQR1L3oMXG0_a_rgeKkdsUbv8EjsBt8j11xNKnTZm5Q,4563
+fractal_server-2.14.0a23.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.0a23.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a23.dist-info/RECORD,,