fractal-server 2.14.4a0__py3-none-any.whl → 2.14.5__py3-none-any.whl

fractal_server/app/runner/executors/slurm_common/remote.py

@@ -1,27 +1,10 @@
-# This adapts clusterfutures <https://github.com/sampsyo/clusterfutures>
-# Original Copyright
-# Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
-# License: MIT
-#
-# Modified by:
-# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-"""
-This module provides a simple self-standing script that executes arbitrary
-python code received via pickled files on a cluster node.
-"""
 import argparse
+import json
 import logging
 import os
 import sys
-from typing import Literal
-from typing import Union
-
-import cloudpickle
 
+from ..call_command_wrapper import call_command_wrapper
 from fractal_server import __VERSION__
 
 
@@ -33,59 +16,6 @@ class FractalVersionMismatch(RuntimeError):
     pass
 
 
-def _check_versions_mismatch(
-    server_versions: dict[
-        Literal["python", "fractal_server", "cloudpickle"],
-        Union[str, tuple[int]],
-    ]
-):
-    """
-    Compare the server {python,cloudpickle,fractal_server} versions with the
-    ones available to the current worker
-
-    Arguments:
-        server_versions:
-            The version used in the fractal-server instance that created the
-            cloudpickle file
-
-    Raises:
-        FractalVersionMismatch: If the cloudpickle or fractal_server versions
-                                do not match with the ones on the server
-    """
-
-    server_python_version = list(server_versions["python"])
-    worker_python_version = list(sys.version_info[:3])
-    if worker_python_version != server_python_version:
-        if worker_python_version[:2] != server_python_version[:2]:
-            # FIXME: Turn this into an error, in some version post 2.14.
-            logging.error(
-                f"{server_python_version=} but {worker_python_version=}. "
-                "This configuration will be deprecated in a future version, "
-                "please contact the admin of this Fractal instance."
-            )
-        else:
-            # Major.minor versions match, patch versions differ
-            logging.warning(
-                f"{server_python_version=} but {worker_python_version=}."
-            )
-
-    server_cloudpickle_version = server_versions["cloudpickle"]
-    worker_cloudpickle_version = cloudpickle.__version__
-    if worker_cloudpickle_version != server_cloudpickle_version:
-        raise FractalVersionMismatch(
-            f"{server_cloudpickle_version=} but "
-            f"{worker_cloudpickle_version=}"
-        )
-
-    server_fractal_server_version = server_versions["fractal_server"]
-    worker_fractal_server_version = __VERSION__
-    if worker_fractal_server_version != server_fractal_server_version:
-        raise FractalVersionMismatch(
-            f"{server_fractal_server_version=} but "
-            f"{worker_fractal_server_version=}"
-        )
-
-
 def worker(
     *,
     in_fname: str,
@@ -95,8 +25,8 @@ def worker(
     Execute a job, possibly on a remote node.
 
     Arguments:
-        in_fname: Absolute path to the input pickle file (must be readable).
-        out_fname: Absolute path of the output pickle file (must be writeable).
+        in_fname: Absolute path to the input file (must be readable).
+        out_fname: Absolute path of the output file (must be writeable).
     """
 
     # Create output folder, if missing
@@ -107,19 +37,49 @@ def worker(
 
     # Execute the job and capture exceptions
     try:
-        with open(in_fname, "rb") as f:
-            indata = f.read()
-        server_versions, fun, args, kwargs = cloudpickle.loads(indata)
-        _check_versions_mismatch(server_versions)
+        with open(in_fname, "r") as f:
+            input_data = json.load(f)
+
+        server_python_version = input_data["python_version"]
+        server_fractal_server_version = input_data["fractal_server_version"]
+
+        # Fractal-server version must be identical
+        worker_fractal_server_version = __VERSION__
+        if worker_fractal_server_version != server_fractal_server_version:
+            raise FractalVersionMismatch(
+                f"{server_fractal_server_version=} but "
+                f"{worker_fractal_server_version=}"
+            )
+
+        # Python version mismatch only raises a warning
+        worker_python_version = tuple(sys.version_info[:3])
+        if worker_python_version != server_python_version:
+            if worker_python_version[:2] != server_python_version[:2]:
+                logging.warning(
+                    f"{server_python_version=} but {worker_python_version=}."
+                )
+
+        # Extract some useful paths
+        metadiff_file_remote = input_data["metadiff_file_remote"]
+        log_path = input_data["log_file_remote"]
+
+        # Execute command
+        full_command = input_data["full_command"]
+        call_command_wrapper(cmd=full_command, log_path=log_path)
+
+        try:
+            with open(metadiff_file_remote, "r") as f:
+                out_meta = json.load(f)
+            result = (True, out_meta)
+        except FileNotFoundError:
+            # Command completed, but it produced no metadiff file
+            result = (True, None)
 
-        result = (True, fun(*args, **kwargs))
-        out = cloudpickle.dumps(result)
     except Exception as e:
         # Exception objects are not serialisable. Here we save the relevant
         # exception contents in a serializable dictionary. Note that whenever
         # the task failed "properly", the exception is a `TaskExecutionError`
         # and it has additional attributes.
-
         import traceback
 
         exc_type, exc_value, traceback_obj = sys.exc_info()
@@ -131,33 +91,28 @@ def worker(
         )
         traceback_string = "".join(traceback_list)
         exc_proxy = dict(
-            exc_type_name=exc_type.__name__,
+            exc_type_name=type(e).__name__,
             traceback_string=traceback_string,
-            workflow_task_order=getattr(e, "workflow_task_order", None),
-            workflow_task_id=getattr(e, "workflow_task_id", None),
-            task_name=getattr(e, "task_name", None),
         )
         result = (False, exc_proxy)
-        out = cloudpickle.dumps(result)
 
-    # Write the output pickle file
-    with open(out_fname, "wb") as f:
-        f.write(out)
+    # Write output file
+    with open(out_fname, "w") as f:
+        json.dump(result, f, indent=2)
 
 
 if __name__ == "__main__":
-
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--input-file",
         type=str,
-        help="Path of input pickle file",
+        help="Path of input JSON file",
         required=True,
     )
     parser.add_argument(
         "--output-file",
         type=str,
-        help="Path of output pickle file",
+        help="Path of output JSON file",
         required=True,
     )
     parsed_args = parser.parse_args()

fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py

@@ -19,48 +19,49 @@ class SlurmTask(BaseModel):
     task_files: TaskFiles
     index: int
 
+    workflow_task_order: int
+    workflow_task_id: int
+    task_name: str
+
     @property
-    def input_pickle_file_local_path(self) -> Path:
+    def input_file_local_path(self) -> Path:
         return (
-            self.workdir_local / f"{self.prefix}-{self.component}-input.pickle"
+            self.workdir_local / f"{self.prefix}-{self.component}-input.json"
         )
 
     @property
-    def input_pickle_file_remote_path(self) -> Path:
+    def input_file_remote_path(self) -> Path:
         return (
-            self.workdir_remote
-            / f"{self.prefix}-{self.component}-input.pickle"
+            self.workdir_remote / f"{self.prefix}-{self.component}-input.json"
         )
 
     @property
-    def output_pickle_file_local_path(self) -> Path:
+    def output_file_local_path(self) -> Path:
         return (
-            self.workdir_local
-            / f"{self.prefix}-{self.component}-output.pickle"
+            self.workdir_local / f"{self.prefix}-{self.component}-output.json"
         )
 
     @property
-    def output_pickle_file_remote_path(self) -> Path:
+    def output_file_remote_path(self) -> Path:
         return (
-            self.workdir_remote
-            / f"{self.prefix}-{self.component}-output.pickle"
+            self.workdir_remote / f"{self.prefix}-{self.component}-output.json"
         )
 
     @property
-    def input_pickle_file_local(self) -> str:
-        return self.input_pickle_file_local_path.as_posix()
+    def input_file_local(self) -> str:
+        return self.input_file_local_path.as_posix()
 
     @property
-    def input_pickle_file_remote(self) -> str:
-        return self.input_pickle_file_remote_path.as_posix()
+    def input_file_remote(self) -> str:
+        return self.input_file_remote_path.as_posix()
 
     @property
-    def output_pickle_file_local(self) -> str:
-        return self.output_pickle_file_local_path.as_posix()
+    def output_file_local(self) -> str:
+        return self.output_file_local_path.as_posix()
 
     @property
-    def output_pickle_file_remote(self) -> str:
-        return self.output_pickle_file_remote_path.as_posix()
+    def output_file_remote(self) -> str:
+        return self.output_file_remote_path.as_posix()
 
 
 class SlurmJob(BaseModel):

fractal_server/app/runner/executors/slurm_ssh/runner.py

@@ -99,9 +99,8 @@ class SlurmSSHRunner(BaseSlurmRunner):
             for task in _slurm_job.tasks:
                 _single_job_filelist.extend(
                     [
-                        task.output_pickle_file_remote_path.name,
+                        task.output_file_remote_path.name,
                         task.task_files.log_file_remote_path.name,
-                        task.task_files.args_file_remote_path.name,
                         task.task_files.metadiff_file_remote_path.name,
                     ]
                 )

fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py

@@ -29,7 +29,6 @@ def _run_command_as_user(
     *,
     cmd: str,
     user: Optional[str] = None,
-    encoding: Optional[str] = "utf-8",
     check: bool = False,
 ) -> subprocess.CompletedProcess:
     """
@@ -38,8 +37,6 @@ def _run_command_as_user(
     Arguments:
         cmd: Command to be run
         user: User to be impersonated
-        encoding: Argument for `subprocess.run`. Note that this must be `None`
-                  to have stdout/stderr as bytes.
         check: If `True`, check that `returncode=0` and fail otherwise.
 
     Raises:
@@ -57,7 +54,7 @@ def _run_command_as_user(
     res = subprocess.run(  # nosec
         shlex.split(new_cmd),
         capture_output=True,
-        encoding=encoding,
+        encoding="utf-8",
     )
     logger.debug(f"[_run_command_as_user] {res.returncode=}")
     logger.debug(f"[_run_command_as_user] {res.stdout=}")

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -102,17 +102,13 @@ class SudoSlurmRunner(BaseSlurmRunner):
             source_target_list.extend(
                 [
                     (
-                        task.output_pickle_file_remote,
-                        task.output_pickle_file_local,
+                        task.output_file_remote,
+                        task.output_file_local,
                     ),
                     (
                         task.task_files.log_file_remote,
                         task.task_files.log_file_local,
                     ),
-                    (
-                        task.task_files.args_file_remote,
-                        task.task_files.args_file_local,
-                    ),
                     (
                         task.task_files.metadiff_file_remote,
                         task.task_files.metadiff_file_local,
@@ -121,17 +117,14 @@ class SudoSlurmRunner(BaseSlurmRunner):
             )
 
         for source, target in source_target_list:
-            # NOTE: By setting encoding=None, we read/write bytes instead
-            # of strings; this is needed to also handle pickle files.
             try:
                 res = _run_command_as_user(
                     cmd=f"cat {source}",
                     user=self.slurm_user,
-                    encoding=None,
                     check=True,
                 )
                 # Write local file
-                with open(target, "wb") as f:
+                with open(target, "w") as f:
                     f.write(res.stdout)
                 logger.debug(
                     f"[_fetch_artifacts_single_job] Copied {source} into "
@@ -171,7 +164,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
         res = _run_command_as_user(
             cmd=cmd,
             user=self.slurm_user,
-            encoding="utf-8",
             check=True,
         )
         return res.stdout

fractal_server/app/runner/task_files.py

@@ -134,14 +134,6 @@ class TaskFiles(BaseModel):
     def metadiff_file_remote(self) -> str:
         return self.metadiff_file_remote_path.as_posix()
 
-    @property
-    def remote_files_dict(self) -> dict[str, str]:
-        return dict(
-            args_file_remote=self.args_file_remote,
-            metadiff_file_remote=self.metadiff_file_remote,
-            log_file_remote=self.log_file_remote,
-        )
-
 
 def enrich_task_files_multisubmit(
     *,

fractal_server/app/runner/v2/_slurm_ssh.py

@@ -13,8 +13,7 @@
 """
 Slurm Backend
 
-This backend runs fractal workflows in a SLURM cluster using Clusterfutures
-Executor objects.
+This backend runs fractal workflows in a SLURM cluster.
 """
 from pathlib import Path
 from typing import Optional

fractal_server/app/runner/v2/_slurm_sudo.py

@@ -13,8 +13,7 @@
 """
 Slurm Backend
 
-This backend runs fractal workflows in a SLURM cluster using Clusterfutures
-Executor objects.
+This backend runs fractal workflows in a SLURM cluster.
 """
 from pathlib import Path
 from typing import Optional

fractal_server/app/runner/v2/runner_functions.py

@@ -1,4 +1,3 @@
-import functools
 from pathlib import Path
 from typing import Any
 from typing import Callable
@@ -12,7 +11,6 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskOutputValidationError
 from .db_tools import update_status_of_history_unit
 from .deduplicate_list import deduplicate_list
-from .runner_functions_low_level import run_single_task
 from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
@@ -207,13 +205,10 @@ def run_v2_task_non_parallel(
         )
 
     result, exception = runner.submit(
-        functools.partial(
-            run_single_task,
-            command=task.command_non_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_non_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         parameters=function_kwargs,
         task_type=task_type,
         task_files=task_files,
@@ -335,13 +330,10 @@ def run_v2_task_parallel(
         )
 
     results, exceptions = runner.multisubmit(
-        functools.partial(
-            run_single_task,
-            command=task.command_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         list_parameters=list_function_kwargs,
         task_type="parallel",
         list_task_files=list_task_files,
@@ -456,13 +448,10 @@ def run_v2_task_compound(
             ],
         )
     result, exception = runner.submit(
-        functools.partial(
-            run_single_task,
-            command=task.command_non_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_non_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         parameters=function_kwargs,
         task_type=task_type,
         task_files=task_files_init,
@@ -559,13 +548,10 @@ def run_v2_task_compound(
         history_unit_ids = [history_unit.id for history_unit in history_units]
 
     results, exceptions = runner.multisubmit(
-        functools.partial(
-            run_single_task,
-            command=task.command_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         list_parameters=list_function_kwargs,
         task_type=task_type,
         list_task_files=list_task_files,

fractal_server/app/runner/versions.py

@@ -2,24 +2,15 @@ import json
 import sys
 from typing import Union
 
-import cloudpickle
-
 import fractal_server
 
 
 def get_versions() -> dict[str, Union[list[int], str]]:
     """
-    Extract versions of Python, fractal-server and cloudpickle.
-
-    This information is useful to check compatibility of two Python
-    interpreters when running tasks: the current interpreter (which prepares
-    the input pickles and orchestrates workflow execution) and a remote
-    interpreter (e.g. the one defined in the `FRACTAL_SLURM_WORKER_PYTHON`
-    configuration variable) that executes the tasks.
+    Extract versions of Python and fractal-server.
     """
     versions = dict(
-        python=list(sys.version_info[:3]),
-        cloudpickle=cloudpickle.__version__,
+        python=tuple(sys.version_info[:3]),
         fractal_server=fractal_server.__VERSION__,
     )
     return versions

fractal_server/config.py

@@ -460,15 +460,6 @@ class Settings(BaseSettings):
     running a task that produces multiple SLURM jobs.
     """
 
-    FRACTAL_SLURM_ERROR_HANDLING_INTERVAL: int = 5
-    """
-    Interval to wait (in seconds) when the SLURM backend does not find an
-    output pickle file - which could be due to several reasons (e.g. the SLURM
-    job was cancelled or failed, or writing the file is taking long). If the
-    file is still missing after this time interval, this leads to a
-    `JobExecutionError`.
-    """
-
     FRACTAL_PIP_CACHE_DIR: Optional[AbsolutePathStr] = None
     """
     Absolute path to the cache directory for `pip`; if unset,

fractal_server/ssh/_fabric.py

@@ -56,6 +56,7 @@ def _acquire_lock_with_timeout(
     """
     logger = get_logger(logger_name)
     logger.info(f"Trying to acquire lock for '{label}', with {timeout=}")
+    t_start_lock_acquire = time.perf_counter()
     result = lock.acquire(timeout=timeout)
     try:
         if not result:
@@ -64,7 +65,9 @@ def _acquire_lock_with_timeout(
                 f"Failed to acquire lock for '{label}' within "
                 f"{timeout} seconds"
             )
-        logger.info(f"Lock for '{label}' was acquired.")
+        t_end_lock_acquire = time.perf_counter()
+        elapsed = t_end_lock_acquire - t_start_lock_acquire
+        logger.info(f"Lock for '{label}' was acquired - {elapsed=:.4f} s")
         yield result
     finally:
         if result:

{fractal_server-2.14.4a0.dist-info → fractal_server-2.14.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.4a0
+Version: 2.14.5
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: alembic (>=1.13.1,<2.0.0)
-Requires-Dist: cloudpickle (>=3.1.0,<3.2.0)
 Requires-Dist: cryptography (>=44.0.1,<44.1.0)
 Requires-Dist: fabric (>=3.2.2,<3.3.0)
 Requires-Dist: fastapi (>=0.115.0,<0.116.0)
@@ -65,8 +64,3 @@ Fractal was conceived in the Liberali Lab at the Friedrich Miescher Institute fo
 
 Unless otherwise specified, Fractal components are released under the BSD 3-Clause License, and copyright is with the BioVisionCenter at the University of Zurich.
 
-The SLURM compatibility layer is based on
-[`clusterfutures`](https://github.com/sampsyo/clusterfutures), by
-[@sampsyo](https://github.com/sampsyo) and collaborators, and it is released
-under the terms of the MIT license.
-