fractal-server 2.17.1a1__py3-none-any.whl → 2.18.0__py3-none-any.whl

fractal_server/runner/config/_local.py

@@ -4,13 +4,16 @@ from pydantic import ConfigDict
 
 class JobRunnerConfigLocal(BaseModel):
     """
-    Specifications of the local-backend configuration
+    Runner-configuration specifications, for a `local` resource.
+
+    The typical use case is that setting `parallel_tasks_per_job` to a
+    small number (e.g. 1) will limit parallelism when executing tasks
+    requiring a large amount of resources (e.g. memory) on a local machine.
 
     Attributes:
         parallel_tasks_per_job:
-            Maximum number of tasks to be run in parallel as part of a call to
-            `FractalThreadPoolExecutor.map`; if `None`, then all tasks will
-            start at the same time.
+            Maximum number of tasks to be run in parallel within a local
+            runner. If `None`, then all tasks may start at the same time.
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -18,4 +21,4 @@ class JobRunnerConfigLocal(BaseModel):
 
     @property
     def batch_size(self) -> int:
-        return self.parallel_tasks_per_job or 1
+        return self.parallel_tasks_per_job or 0

fractal_server/runner/config/_slurm.py

@@ -10,13 +10,15 @@ from fractal_server.runner.config.slurm_mem_to_MB import slurm_mem_to_MB
 from fractal_server.types import DictStrStr
 from fractal_server.types import NonEmptyStr
 
-
 MemMBType = Annotated[
     PositiveInt | NonEmptyStr, AfterValidator(slurm_mem_to_MB)
 ]
+"""
+Memory expressed in MB.
+"""
 
 
-class _SlurmConfigSet(BaseModel):
+class SlurmConfigSet(BaseModel):
     """
     Options for the default or gpu SLURM config.
 
@@ -24,7 +26,6 @@ class _SlurmConfigSet(BaseModel):
         partition:
         cpus_per_task:
         mem:
-            See `_parse_mem_value` for details on allowed values.
         constraint:
         gres:
         time:
@@ -32,6 +33,8 @@ class _SlurmConfigSet(BaseModel):
         nodelist:
         account:
         extra_lines:
+        gpus:
+        shebang_line: The shell shebang to use for SLURM jobs.
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -47,9 +50,10 @@ class _SlurmConfigSet(BaseModel):
     account: NonEmptyStr | None = None
     extra_lines: list[NonEmptyStr] = Field(default_factory=list)
     gpus: NonEmptyStr | None = None
+    shebang_line: str = "#!/bin/sh"
 
 
-class _BatchingConfigSet(BaseModel):
+class BatchingConfigSet(BaseModel):
     """
     Options to configure the batching strategy (that is, how to combine
     several tasks in a single SLURM job).
@@ -58,9 +62,7 @@ class _BatchingConfigSet(BaseModel):
         target_cpus_per_job:
         max_cpus_per_job:
         target_mem_per_job:
-            (see `_parse_mem_value` for details on allowed values)
         max_mem_per_job:
-            (see `_parse_mem_value` for details on allowed values)
         target_num_jobs:
         max_num_jobs:
     """
@@ -77,34 +79,38 @@ class _BatchingConfigSet(BaseModel):
 
 class JobRunnerConfigSLURM(BaseModel):
     """
-    Common SLURM configuration.
+    Runner-configuration specifications, for a `slurm_sudo` or
+    `slurm_ssh` resource.
 
-    Note: this is a common and abstract class, which gets transformed into
-    more specific configuration objects during job execution.
+    Note: this is a common class, which is processed and transformed into more
+    specific configuration objects during job execution.
 
     Valid JSON example
-    ```JSON
+    ```json
     {
-      "default_slurm_config": {
-          "partition": "main",
-          "cpus_per_task": 1
-      },
-      "gpu_slurm_config": {
-          "partition": "gpu",
-          "extra_lines": ["#SBATCH --gres=gpu:v100:1"]
-      },
-      "batching_config": {
-          "target_cpus_per_job": 1,
-          "max_cpus_per_job": 1,
-          "target_mem_per_job": 200,
-          "max_mem_per_job": 500,
-          "target_num_jobs": 2,
-          "max_num_jobs": 4
-      },
-      "user_local_exports": {
-          "CELLPOSE_LOCAL_MODELS_PATH": "CELLPOSE_LOCAL_MODELS_PATH",
-          "NAPARI_CONFIG": "napari_config.json"
-      }
+        "default_slurm_config": {
+            "partition": "partition-name",
+            "cpus_per_task": 1,
+            "mem": "100M"
+        },
+        "gpu_slurm_config": {
+            "partition": "gpu",
+            "extra_lines": [
+                "#SBATCH --gres=gpu:v100:1"
+            ]
+        },
+        "user_local_exports": {
+            "CELLPOSE_LOCAL_MODELS_PATH": "CELLPOSE_LOCAL_MODELS_PATH",
+            "NAPARI_CONFIG": "napari_config.json"
+        },
+        "batching_config": {
+            "target_cpus_per_job": 1,
+            "max_cpus_per_job": 1,
+            "target_mem_per_job": 200,
+            "max_mem_per_job": 500,
+            "target_num_jobs": 2,
+            "max_num_jobs": 4
+        }
     }
     ```
 
@@ -123,7 +129,7 @@ class JobRunnerConfigSLURM(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    default_slurm_config: _SlurmConfigSet
-    gpu_slurm_config: _SlurmConfigSet | None = None
-    batching_config: _BatchingConfigSet
+    default_slurm_config: SlurmConfigSet
+    gpu_slurm_config: SlurmConfigSet | None = None
+    batching_config: BatchingConfigSet
     user_local_exports: DictStrStr = Field(default_factory=dict)

fractal_server/runner/config/slurm_mem_to_MB.py

@@ -1,7 +1,6 @@
 from fractal_server.logger import set_logger
 from fractal_server.runner.exceptions import SlurmConfigError
 
-
 logger = set_logger(__name__)
 
 

fractal_server/runner/executors/base_runner.py

@@ -9,13 +9,32 @@ from fractal_server.runner.task_files import TaskFiles
 
 
 class SubmitTaskType(StrEnum):
-    COMPOUND = TaskType.COMPOUND
+    """
+    Valid task types for `BaseRunner.submit`.
+
+    Attributes:
+        NON_PARALLEL: Non-parallel task.
+        COMPOUND: Compound task.
+        CONVERTER_NON_PARALLEL: Non-parallel converter task.
+        CONVERTER_COMPOUND: Compound converter task.
+    """
+
     NON_PARALLEL = TaskType.NON_PARALLEL
+    COMPOUND = TaskType.COMPOUND
     CONVERTER_NON_PARALLEL = TaskType.CONVERTER_NON_PARALLEL
     CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND
 
 
 class MultisubmitTaskType(StrEnum):
+    """
+    Valid task types for `BaseRunner.multisubmit`.
+
+    Attributes:
+        PARALLEL: Parallel task.
+        COMPOUND: Compound task.
+        CONVERTER_COMPOUND: Compound converter task.
+    """
+
     PARALLEL = TaskType.PARALLEL
     COMPOUND = TaskType.COMPOUND
     CONVERTER_COMPOUND = TaskType.CONVERTER_COMPOUND
@@ -27,11 +46,18 @@ TASK_TYPES_SUBMIT: list[TaskType] = [
     TaskType.NON_PARALLEL,
     TaskType.CONVERTER_NON_PARALLEL,
 ]
+"""
+List of valid task types for `BaseRunner.submit`.
+"""
+
 TASK_TYPES_MULTISUBMIT: list[TaskType] = [
     TaskType.COMPOUND,
     TaskType.CONVERTER_COMPOUND,
     TaskType.PARALLEL,
 ]
+"""
+List of valid task types for `BaseRunner.multisubmit`.
+"""
 
 logger = set_logger(__name__)
 
@@ -125,7 +151,7 @@ class BaseRunner:
 
         Args:
             parameters: Parameters dictionary.
-            task_type: Task type.s
+            task_type: Task type.
         """
         logger.info("[validate_submit_parameters] START")
         if task_type not in TASK_TYPES_SUBMIT:
@@ -175,8 +201,7 @@ class BaseRunner:
 
         if len(list_parameters) != len(list_task_files):
             raise ValueError(
-                f"{len(list_task_files)=} differs from "
-                f"{len(list_parameters)=}."
+                f"{len(list_task_files)=} differs from {len(list_parameters)=}."
             )
         if len(history_unit_ids) != len(list_parameters):
             raise ValueError(

fractal_server/runner/executors/local/get_local_config.py

@@ -1,6 +1,7 @@
 """
 Submodule to handle the local-backend configuration for a WorkflowTask
 """
+
 from typing import Literal
 
 from fractal_server.app.models.v2 import WorkflowTaskV2

fractal_server/runner/executors/local/runner.py

@@ -4,7 +4,6 @@ from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
 
-from ..call_command_wrapper import call_command_wrapper
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models import Profile
 from fractal_server.app.models import Resource
@@ -16,10 +15,11 @@ from fractal_server.runner.exceptions import TaskExecutionError
 from fractal_server.runner.executors.base_runner import BaseRunner
 from fractal_server.runner.executors.base_runner import MultisubmitTaskType
 from fractal_server.runner.executors.base_runner import SubmitTaskType
-from fractal_server.runner.task_files import TaskFiles
-from fractal_server.runner.v2.db_tools import (
-    bulk_update_status_of_history_unit,
+from fractal_server.runner.executors.call_command_wrapper import (
+    call_command_wrapper,
 )
+from fractal_server.runner.task_files import TaskFiles
+from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
 from fractal_server.runner.v2.db_tools import update_status_of_history_unit
 
 logger = set_logger(__name__)
@@ -56,6 +56,13 @@ def run_single_task(
 
 
 class LocalRunner(BaseRunner):
+    """
+    Runner implementation for a computational `local` resource.
+
+    Tasks are executed through a `concurrent.futures.ThreadPoolExecutor`
+    executor.
+    """
+
     executor: ThreadPoolExecutor
     root_dir_local: Path
     shared_config: JobRunnerConfigLocal
@@ -70,9 +77,7 @@ class LocalRunner(BaseRunner):
         self.root_dir_local.mkdir(parents=True, exist_ok=True)
         self.executor = ThreadPoolExecutor()
         logger.debug("Create LocalRunner")
-        self.shared_config = JobRunnerConfigLocal(
-            **resource.jobs_runner_config
-        )
+        self.shared_config = JobRunnerConfigLocal(**resource.jobs_runner_config)
 
     def __enter__(self):
         logger.debug("Enter LocalRunner")
@@ -235,9 +240,7 @@ class LocalRunner(BaseRunner):
                 f"Original error {str(e)}"
             )
             exception = TaskExecutionError(str(e))
-            exceptions = {
-                ind: exception for ind in range(len(list_parameters))
-            }
+            exceptions = {ind: exception for ind in range(len(list_parameters))}
             if task_type == TaskType.PARALLEL:
                 with next(get_sync_db()) as db:
                     bulk_update_status_of_history_unit(
@@ -269,9 +272,7 @@ class LocalRunner(BaseRunner):
                         "[multisubmit] Unexpected exception during submission."
                         f" Original error {str(e)}"
                     )
-                    current_history_unit_id = history_unit_ids[
-                        positional_index
-                    ]
+                    current_history_unit_id = history_unit_ids[positional_index]
                     exceptions[positional_index] = TaskExecutionError(str(e))
                     if task_type == TaskType.PARALLEL:
                         with next(get_sync_db()) as db:

fractal_server/runner/executors/slurm_common/_batching.py

@@ -1,16 +1,7 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
 """
 Submodule to determine the number of total/parallel tasks per SLURM job.
 """
+
 import math
 
 from fractal_server.logger import set_logger
@@ -19,6 +10,10 @@ logger = set_logger(__name__)
 
 
 class SlurmHeuristicsError(ValueError):
+    """
+    Error in SLURM-batching heuristics.
+    """
+
     pass
 
 
@@ -130,16 +125,12 @@ def heuristics(
         raise SlurmHeuristicsError(msg)
     if cpus_per_task > max_cpus_per_job:
         msg = (
-            f"[heuristics] Requested {cpus_per_task=} "
-            f"but {max_cpus_per_job=}."
+            f"[heuristics] Requested {cpus_per_task=} but {max_cpus_per_job=}."
         )
         logger.error(msg)
         raise SlurmHeuristicsError(msg)
     if mem_per_task > max_mem_per_job:
-        msg = (
-            f"[heuristics] Requested {mem_per_task=} "
-            f"but {max_mem_per_job=}."
-        )
+        msg = f"[heuristics] Requested {mem_per_task=} but {max_mem_per_job=}."
         logger.error(msg)
         raise SlurmHeuristicsError(msg)
 
@@ -177,8 +168,7 @@ def heuristics(
             )
         if mem_per_job > max_mem_per_job:
             msg = (
-                f"[heuristics] Requested {mem_per_job=} "
-                f"but {max_mem_per_job=}."
+                f"[heuristics] Requested {mem_per_job=} but {max_mem_per_job=}."
             )
             logger.error(msg)
             raise SlurmHeuristicsError(msg)
@@ -187,8 +177,7 @@ def heuristics(
         num_jobs = math.ceil(tot_tasks / tasks_per_job)
         if num_jobs > target_num_jobs:
             logger.debug(
-                f"[heuristics] Requested {num_jobs=} "
-                f"but {target_num_jobs=}."
+                f"[heuristics] Requested {num_jobs=} but {target_num_jobs=}."
             )
         if num_jobs > max_num_jobs:
             msg = f"[heuristics] Requested {num_jobs=} but {max_num_jobs=}."

fractal_server/runner/executors/slurm_common/base_slurm_runner.py

@@ -9,10 +9,6 @@ from typing import Literal
 from pydantic import BaseModel
 from pydantic import ConfigDict
 
-from ..slurm_common.slurm_job_task_models import SlurmJob
-from ..slurm_common.slurm_job_task_models import SlurmTask
-from ._job_states import STATES_FINISHED
-from .slurm_config import SlurmConfig
 from fractal_server import __VERSION__
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import AccountingRecordSlurm
@@ -25,15 +21,47 @@ from fractal_server.runner.exceptions import TaskExecutionError
 from fractal_server.runner.executors.base_runner import BaseRunner
 from fractal_server.runner.executors.base_runner import MultisubmitTaskType
 from fractal_server.runner.executors.base_runner import SubmitTaskType
+from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
+    SlurmJob,
+)
+from fractal_server.runner.executors.slurm_common.slurm_job_task_models import (
+    SlurmTask,
+)
 from fractal_server.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.runner.task_files import TaskFiles
-from fractal_server.runner.v2.db_tools import (
-    bulk_update_status_of_history_unit,
-)
+from fractal_server.runner.v2.db_tools import bulk_update_status_of_history_unit
 from fractal_server.runner.v2.db_tools import update_status_of_history_unit
 
+from ._job_states import STATES_FINISHED
+from .slurm_config import SlurmConfig
+
 SHUTDOWN_ERROR_MESSAGE = "Failed due to job-execution shutdown."
 SHUTDOWN_EXCEPTION = JobExecutionError(SHUTDOWN_ERROR_MESSAGE)
+STDERR_IGNORE_PATTERNS = [
+    "step creation temporarily disabled, retrying",
+    "step creation still disabled, retrying",
+    "srun: step created for stepid=",
+]
+
+
+def ignore_stderr_line(line: str) -> bool:
+    """
+    Whether to ignore a SLURM-job stderr line, based on
+    `STDERR_EXCLUDE_PATTERNS`.
+
+    The goal is not to flag some stderr files as relevant `executor_error_log`
+    if they only include lines matching a given set of patterns to ignore. See
+    https://github.com/fractal-analytics-platform/fractal-server/issues/2835
+
+    Args:
+        line: The line to be considered.
+    """
+    line_lower = line.lower()
+    for pattern in STDERR_IGNORE_PATTERNS:
+        if pattern in line_lower:
+            return True
+    return False
+
 
 logger = set_logger(__name__)
 
@@ -65,6 +93,10 @@ def create_accounting_record_slurm(
 
 
 class BaseSlurmRunner(BaseRunner):
+    """
+    Base class for SLURM runners.
+    """
+
     shutdown_file: Path
     common_script_lines: list[str]
     user_cache_dir: str
@@ -307,8 +339,7 @@ class BaseSlurmRunner(BaseRunner):
                 json.dump(task.parameters, f, indent=2)
 
             logger.debug(
-                "[_prepare_single_slurm_job] Written "
-                f"{task.input_file_local=}"
+                f"[_prepare_single_slurm_job] Written {task.input_file_local=}"
             )
 
         # Prepare commands to be included in SLURM submission script
@@ -352,9 +383,7 @@ class BaseSlurmRunner(BaseRunner):
         # Always print output of `uname -n` and `pwd`
         script_lines.append('\necho "Hostname: $(uname -n)"')
         script_lines.append('echo "Current directory: $(pwd)"')
-        script_lines.append(
-            'echo "Start time: $(date +"%Y-%m-%dT%H:%M:%S%z")"'
-        )
+        script_lines.append('echo "Start time: $(date +"%Y-%m-%dT%H:%M:%S%z")"')
 
         # Complete script preamble
         script_lines.append("\n")
@@ -368,9 +397,7 @@ class BaseSlurmRunner(BaseRunner):
                 f"{cmd} &"
             )
         script_lines.append("wait\n\n")
-        script_lines.append(
-            'echo "End time:   $(date +"%Y-%m-%dT%H:%M:%S%z")"'
-        )
+        script_lines.append('echo "End time:   $(date +"%Y-%m-%dT%H:%M:%S%z")"')
         script = "\n".join(script_lines)
 
         # Write submission script
@@ -528,7 +555,13 @@ class BaseSlurmRunner(BaseRunner):
 
         try:
             with open(stderr_path) as f:
-                stderr_content = f.read().strip()
+                stderr_lines = [
+                    line
+                    for line in f.readlines()
+                    if not ignore_stderr_line(line)
+                ]
+                stderr_content = "\n".join(stderr_lines)
+                stderr_content = stderr_content.strip()
             if stderr_content:
                 return stderr_content
         except Exception as e:
@@ -595,8 +628,7 @@ class BaseSlurmRunner(BaseRunner):
     def _check_no_active_jobs(self):
         if self.jobs != {}:
             raise JobExecutionError(
-                "Unexpected branch: jobs must be empty before new "
-                "submissions."
+                "Unexpected branch: jobs must be empty before new submissions."
             )
 
     def submit(
@@ -711,9 +743,7 @@ class BaseSlurmRunner(BaseRunner):
             scancelled_job_ids = []
             while len(self.jobs) > 0:
                 # Look for finished jobs
-                finished_job_ids = self._get_finished_jobs(
-                    job_ids=self.job_ids
-                )
+                finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
                 logger.debug(f"[submit] {finished_job_ids=}")
                 finished_jobs = [
                     self.jobs[_slurm_job_id]
@@ -860,9 +890,7 @@ class BaseSlurmRunner(BaseRunner):
                 args_batches.append(
                     list_parameters[ind_chunk : ind_chunk + batch_size]  # noqa
                 )
-            if len(args_batches) != math.ceil(
-                tot_tasks / config.tasks_per_job
-            ):
+            if len(args_batches) != math.ceil(tot_tasks / config.tasks_per_job):
                 raise RuntimeError("Something wrong here while batching tasks")
 
             # Part 1/3: Iterate over chunks, prepare SlurmJob objects
@@ -975,9 +1003,7 @@ class BaseSlurmRunner(BaseRunner):
                     logger.debug(f"[multisubmit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
                     for task in slurm_job.tasks:
-                        logger.debug(
-                            f"[multisubmit] Now process {task.index=}"
-                        )
+                        logger.debug(f"[multisubmit] Now process {task.index=}")
                         was_job_scancelled = slurm_job_id in scancelled_job_ids
                         if fetch_artifacts_exception is not None:
                             result = None

fractal_server/runner/executors/slurm_common/get_slurm_config.py

@@ -1,14 +1,15 @@
 from typing import Literal
 
-from ._batching import heuristics
-from .slurm_config import logger
-from .slurm_config import SlurmConfig
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.runner.config import JobRunnerConfigSLURM
 from fractal_server.runner.config.slurm_mem_to_MB import slurm_mem_to_MB
 from fractal_server.runner.exceptions import SlurmConfigError
 from fractal_server.string_tools import interpret_as_bool
 
+from ._batching import heuristics
+from .slurm_config import SlurmConfig
+from .slurm_config import logger
+
 
 def _get_slurm_config_internal(
     shared_config: JobRunnerConfigSLURM,
@@ -33,7 +34,6 @@ def _get_slurm_config_internal(
         which_type:
             Whether we should look at the non-parallel or parallel part
             of `wftask`.
-        tot_tasks: Not used here, only present as a common interface.
 
     Returns:
         A ready-to-use `SlurmConfig` object.
@@ -138,9 +138,7 @@ def _get_slurm_config_internal(
     # Job-batching parameters (if None, they will be determined heuristically)
     if wftask_meta is not None:
         tasks_per_job = wftask_meta.get("tasks_per_job", None)
-        parallel_tasks_per_job = wftask_meta.get(
-            "parallel_tasks_per_job", None
-        )
+        parallel_tasks_per_job = wftask_meta.get("parallel_tasks_per_job", None)
     else:
         tasks_per_job = None
         parallel_tasks_per_job = None
@@ -162,6 +160,15 @@ def get_slurm_config(
     which_type: Literal["non_parallel", "parallel"],
     tot_tasks: int = 1,
 ) -> SlurmConfig:
+    """
+    Get `SlurmConfig` object.
+
+    Args:
+        shared_config:
+        wftask:
+        which_type:
+        tot_tasks:
+    """
     config = _get_slurm_config_internal(
         shared_config=shared_config,
         wftask=wftask,

fractal_server/runner/executors/slurm_common/remote.py

@@ -3,8 +3,10 @@ import json
 import os
 import sys
 
-from ..call_command_wrapper import call_command_wrapper
 from fractal_server import __VERSION__
+from fractal_server.runner.executors.call_command_wrapper import (
+    call_command_wrapper,
+)
 
 
 class FractalVersionMismatch(RuntimeError):

fractal_server/runner/executors/slurm_common/slurm_config.py

@@ -1,6 +1,7 @@
 """
 Submodule to handle the SLURM configuration for a WorkflowTask
 """
+
 from pathlib import Path
 
 from pydantic import BaseModel
@@ -59,6 +60,7 @@ class SlurmConfig(BaseModel):
             Key-value pairs to be included as `export`-ed variables in SLURM
             submission script, after prepending values with the user's cache
             directory.
+
     """
 
     model_config = ConfigDict(extra="forbid")

fractal_server/runner/executors/slurm_common/slurm_job_task_models.py

@@ -24,9 +24,7 @@ class SlurmTask(BaseModel):
 
     @property
     def input_file_local_path(self) -> Path:
-        return (
-            self.workdir_local / f"{self.prefix}-{self.component}-input.json"
-        )
+        return self.workdir_local / f"{self.prefix}-{self.component}-input.json"
 
     @property
     def input_file_remote_path(self) -> Path: