fractal-server 2.16.5__py3-none-any.whl → 2.17.0a0__py3-none-any.whl

fractal_server/runner/executors/slurm_common/slurm_job_task_models.py

@@ -112,7 +112,7 @@ class SlurmJob(BaseModel):
         return self.slurm_stderr_remote_path.as_posix()
 
     @property
-    def slurm_stdout_local_path(self) -> str:
+    def slurm_stdout_local_path(self) -> Path:
         return (
             self.workdir_local
             / f"{self.prefix}-slurm-{self.slurm_job_id_placeholder}.out"

fractal_server/runner/executors/slurm_ssh/runner.py

@@ -6,13 +6,13 @@ from ..slurm_common.slurm_job_task_models import SlurmJob
 from .run_subprocess import run_subprocess
 from .tar_commands import get_tar_compression_cmd
 from .tar_commands import get_tar_extraction_cmd
-from fractal_server.config import get_settings
+from fractal_server.app.models import Profile
+from fractal_server.app.models import Resource
 from fractal_server.logger import set_logger
+from fractal_server.runner.config import JobRunnerConfigSLURM
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.ssh._fabric import FractalSSHCommandError
 from fractal_server.ssh._fabric import FractalSSHTimeoutError
-from fractal_server.syringe import Inject
-
 
 logger = set_logger(__name__)
 
@@ -27,10 +27,11 @@ class SlurmSSHRunner(BaseSlurmRunner):
         root_dir_local: Path,
         root_dir_remote: Path,
         common_script_lines: list[str] | None = None,
-        user_cache_dir: str | None = None,
-        poll_interval: int | None = None,
+        resource: Resource,
         # Specific
         slurm_account: str | None = None,
+        profile: Profile,
+        user_cache_dir: str | None = None,
         fractal_ssh: FractalSSH,
     ) -> None:
         """
@@ -38,20 +39,21 @@ class SlurmSSHRunner(BaseSlurmRunner):
         different SLURM jobs/tasks.
         """
         self.fractal_ssh = fractal_ssh
+        self.shared_config = JobRunnerConfigSLURM(
+            **resource.jobs_runner_config
+        )
         logger.warning(self.fractal_ssh)
 
         # Check SSH connection and try to recover from a closed-socket error
         self.fractal_ssh.check_connection()
-
-        settings = Inject(get_settings)
         super().__init__(
             slurm_runner_type="ssh",
             root_dir_local=root_dir_local,
             root_dir_remote=root_dir_remote,
             common_script_lines=common_script_lines,
             user_cache_dir=user_cache_dir,
-            poll_interval=poll_interval,
-            python_worker_interpreter=settings.FRACTAL_SLURM_WORKER_PYTHON,
+            poll_interval=resource.jobs_poll_interval,
+            python_worker_interpreter=resource.jobs_slurm_python_worker,
             slurm_account=slurm_account,
         )
 
@@ -225,11 +227,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
 
         logger.debug("[_send_many_job_inputs] END.")
 
-    def run_squeue(
-        self,
-        *,
-        job_ids: list[str],
-    ) -> str:
+    def run_squeue(self, *, job_ids: list[str]) -> str:
         """
         Run `squeue` for a set of SLURM job IDs.
 

fractal_server/runner/executors/slurm_sudo/_subprocess_run_as_user.py

@@ -33,7 +33,7 @@ def _run_command_as_user(
     """
     Use `sudo -u` to impersonate another user and run a command
 
-    Arguments:
+    Args:
         cmd: Command to be run
         user: User to be impersonated
         check: If `True`, check that `returncode=0` and fail otherwise.
@@ -71,7 +71,7 @@ def _mkdir_as_user(*, folder: str, user: str) -> None:
     """
     Create a folder as a different user
 
-    Arguments:
+    Args:
         folder: Absolute path to the folder
         user: User to be impersonated
 

fractal_server/runner/executors/slurm_sudo/runner.py

@@ -2,7 +2,6 @@ import logging
 import os
 import shlex
 import subprocess  # nosec
-import sys
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
@@ -10,10 +9,11 @@ from ..slurm_common.base_slurm_runner import BaseSlurmRunner
 from ..slurm_common.slurm_job_task_models import SlurmJob
 from ._subprocess_run_as_user import _mkdir_as_user
 from ._subprocess_run_as_user import _run_command_as_user
-from fractal_server.config import get_settings
+from fractal_server.app.models import Profile
+from fractal_server.app.models import Resource
 from fractal_server.logger import set_logger
+from fractal_server.runner.config import JobRunnerConfigSLURM
 from fractal_server.runner.exceptions import JobExecutionError
-from fractal_server.syringe import Inject
 
 logger = set_logger(__name__)
 
@@ -51,19 +51,21 @@ class SudoSlurmRunner(BaseSlurmRunner):
         root_dir_local: Path,
         root_dir_remote: Path,
         common_script_lines: list[str] | None = None,
-        user_cache_dir: str | None = None,
-        poll_interval: int | None = None,
+        resource: Resource,
         # Specific
+        profile: Profile,
+        user_cache_dir: str | None = None,  # FIXME: make required?
         slurm_account: str | None = None,
-        slurm_user: str,
     ) -> None:
         """
         Set parameters that are the same for different Fractal tasks and for
         different SLURM jobs/tasks.
         """
 
-        self.slurm_user = slurm_user
-        settings = Inject(get_settings)
+        self.slurm_user = profile.username
+        self.shared_config = JobRunnerConfigSLURM(
+            **resource.jobs_runner_config
+        )
 
         super().__init__(
             slurm_runner_type="sudo",
@@ -71,10 +73,8 @@ class SudoSlurmRunner(BaseSlurmRunner):
             root_dir_remote=root_dir_remote,
             common_script_lines=common_script_lines,
             user_cache_dir=user_cache_dir,
-            poll_interval=poll_interval,
-            python_worker_interpreter=(
-                settings.FRACTAL_SLURM_WORKER_PYTHON or sys.executable
-            ),
+            poll_interval=resource.jobs_poll_interval,
+            python_worker_interpreter=resource.jobs_slurm_python_worker,
             slurm_account=slurm_account,
         )
 

fractal_server/runner/v2/_local.py

@@ -5,16 +5,19 @@ from ..executors.local.runner import LocalRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
 from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import Profile
+from fractal_server.app.models.v2 import Resource
 from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.types import AttributeFilters
 
 
 def process_workflow(
     *,
+    job_id: int,
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
-    job_id: int,
     workflow_dir_remote: Path | None = None,
     first_task_index: int | None = None,
     last_task_index: int | None = None,
@@ -22,23 +25,24 @@ def process_workflow(
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
-    **kwargs,
+    resource: Resource,
+    profile: Profile,
+    user_cache_dir: str | None = None,
+    fractal_ssh: FractalSSH | None = None,
+    slurm_account: str | None = None,
+    worker_init: str | None = None,
 ) -> None:
     """
-    Run a workflow through
+    Run a workflow through a local backend.
 
     Args:
-        workflow:
-            The workflow to be run
-        dataset:
-            Initial dataset.
-        workflow_dir_local:
-            Working directory for this run.
+        job_id: Job ID.
+        workflow: Workflow to be run
+        dataset: Dataset to be used.
+        workflow_dir_local: Local working directory for this job.
         workflow_dir_remote:
-            Working directory for this run, on the user side. This argument is
-            present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from
-            `workflow_dir_local`.
+            Remote working directory for this job - only relevant for
+            `slurm_sudo` and `slurm_ssh` backends.
         first_task_index:
             Positional index of the first task to execute; if `None`, start
             from `0`.
@@ -46,13 +50,20 @@ def process_workflow(
             Positional index of the last task to execute; if `None`, proceed
             until the last task.
         logger_name: Logger name
-        user_id:
-
-    Raises:
-        TaskExecutionError: wrapper for errors raised during tasks' execution
-                            (positive exit codes).
-        JobExecutionError: wrapper for errors raised by the tasks' executors
-                           (negative exit codes).
+        user_id: User ID.
+        resource: Computational resource for running this job.
+        profile: Computational profile for running this job.
+        user_cache_dir:
+            User-writeable folder (typically a subfolder of `project_dir`).
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        fractal_ssh:
+            `FractalSSH` object, only relevant for the `slurm_ssh` backend.
+        slurm_account:
+            SLURM account to set.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        worker_init:
+            Additional preamble lines for SLURM submission script.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -69,7 +80,11 @@ def process_workflow(
         last_task_index=last_task_index,
     )
 
-    with LocalRunner(root_dir_local=workflow_dir_local) as runner:
+    with LocalRunner(
+        root_dir_local=workflow_dir_local,
+        resource=resource,
+        profile=profile,
+    ) as runner:
         execute_tasks_v2(
             wf_task_list=workflow.task_list[
                 first_task_index : (last_task_index + 1)

fractal_server/runner/v2/_slurm_ssh.py

@@ -23,6 +23,8 @@ from ..executors.slurm_ssh.runner import SlurmSSHRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
 from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import Profile
+from fractal_server.app.models.v2 import Resource
 from fractal_server.app.models.v2 import WorkflowV2
 from fractal_server.logger import set_logger
 from fractal_server.types import AttributeFilters
@@ -32,10 +34,10 @@ logger = set_logger(__name__)
 
 def process_workflow(
     *,
+    job_id: int,
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
-    job_id: int,
     workflow_dir_remote: Path | None = None,
     first_task_index: int | None = None,
     last_task_index: int | None = None,
@@ -43,13 +45,45 @@ def process_workflow(
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
-    # SLURM-ssh-specific
-    fractal_ssh: FractalSSH,
+    resource: Resource,
+    profile: Profile,
+    fractal_ssh: FractalSSH | None = None,
     slurm_account: str | None = None,
     worker_init: str | None = None,
+    user_cache_dir: str | None = None,
 ) -> None:
     """
-    Process workflow (SLURM backend public interface)
+    Run a workflow through a `slurm_ssh` backend.
+
+        Args:
+        job_id: Job ID.
+        workflow: Workflow to be run
+        dataset: Dataset to be used.
+        workflow_dir_local: Local working directory for this job.
+        workflow_dir_remote:
+            Remote working directory for this job - only relevant for
+            `slurm_sudo` and `slurm_ssh` backends.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+        logger_name: Logger name
+        user_id: User ID.
+        resource: Computational resource for running this job.
+        profile: Computational profile for running this job.
+        user_cache_dir:
+            User-writeable folder (typically a subfolder of `project_dir`).
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        fractal_ssh:
+            `FractalSSH` object, only relevant for the `slurm_ssh` backend.
+        slurm_account:
+            SLURM account to set.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        worker_init:
+            Additional preamble lines for SLURM submission script.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
     """
 
     # Set values of first_task_index and last_task_index
@@ -68,6 +102,8 @@ def process_workflow(
         root_dir_local=workflow_dir_local,
         root_dir_remote=workflow_dir_remote,
         slurm_account=slurm_account,
+        resource=resource,
+        profile=profile,
         common_script_lines=worker_init,
     ) as runner:
         execute_tasks_v2(

fractal_server/runner/v2/_slurm_sudo.py

@@ -22,16 +22,19 @@ from ..executors.slurm_sudo.runner import SudoSlurmRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
 from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import Profile
+from fractal_server.app.models.v2 import Resource
 from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.types import AttributeFilters
 
 
 def process_workflow(
     *,
+    job_id: int,
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
-    job_id: int,
     workflow_dir_remote: Path | None = None,
     first_task_index: int | None = None,
     last_task_index: int | None = None,
@@ -39,14 +42,45 @@ def process_workflow(
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
-    # SLURM-sudo-specific
+    resource: Resource,
+    profile: Profile,
     user_cache_dir: str | None = None,
-    slurm_user: str | None = None,
     slurm_account: str | None = None,
     worker_init: str | None = None,
+    fractal_ssh: FractalSSH | None = None,
 ) -> None:
     """
-    Process workflow (SLURM backend public interface).
+    Run a workflow through a `slurm_sudo` backend.
+
+        Args:
+        job_id: Job ID.
+        workflow: Workflow to be run
+        dataset: Dataset to be used.
+        workflow_dir_local: Local working directory for this job.
+        workflow_dir_remote:
+            Remote working directory for this job - only relevant for
+            `slurm_sudo` and `slurm_ssh` backends.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+        logger_name: Logger name
+        user_id: User ID.
+        resource: Computational resource for running this job.
+        profile: Computational profile for running this job.
+        user_cache_dir:
+            User-writeable folder (typically a subfolder of `project_dir`).
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        fractal_ssh:
+            `FractalSSH` object, only relevant for the `slurm_ssh` backend.
+        slurm_account:
+            SLURM account to set.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
+        worker_init:
+            Additional preamble lines for SLURM submission script.
+            Only relevant for `slurm_sudo` and `slurm_ssh` backends.
     """
 
     # Set values of first_task_index and last_task_index
@@ -57,20 +91,16 @@ def process_workflow(
         last_task_index=last_task_index,
     )
 
-    if not slurm_user:
-        raise RuntimeError(
-            "slurm_user argument is required, for slurm backend"
-        )
-
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
     with SudoSlurmRunner(
-        slurm_user=slurm_user,
-        user_cache_dir=user_cache_dir,
         root_dir_local=workflow_dir_local,
         root_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
+        resource=resource,
+        profile=profile,
+        user_cache_dir=user_cache_dir,
         slurm_account=slurm_account,
     ) as runner:
         execute_tasks_v2(

fractal_server/runner/v2/db_tools.py

@@ -88,7 +88,7 @@ def bulk_upsert_image_cache_fast(
     NOTE: we tried to replace `index_elements` with
     `constraint="pk_historyimagecache"`, but it did not work as expected.
 
-    Arguments:
+    Args:
         list_upsert_objects:
             List of dictionaries for objects to be upsert-ed.
         db: A sync database session

fractal_server/runner/v2/runner.py

@@ -1,15 +1,14 @@
-from collections.abc import Callable
 from copy import copy
 from copy import deepcopy
 from pathlib import Path
 from typing import Any
-from typing import Literal
 
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import delete
 from sqlmodel import update
 
 from .merge_outputs import merge_outputs
+from .runner_functions import GetRunnerConfigType
 from .runner_functions import run_v2_task_compound
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
@@ -92,14 +91,7 @@ def execute_tasks_v2(
     job_id: int,
     workflow_dir_remote: Path | None = None,
     logger_name: str | None = None,
-    get_runner_config: Callable[
-        [
-            WorkflowTaskV2,
-            Literal["non_parallel", "parallel"],
-            Path | None,
-        ],
-        Any,
-    ],
+    get_runner_config: GetRunnerConfigType,
     job_type_filters: dict[str, bool],
     job_attribute_filters: AttributeFilters,
 ) -> None:
@@ -257,7 +249,7 @@ def execute_tasks_v2(
         except Exception as e:
             outcomes_dict = {
                 0: SubmissionOutcome(
-                    result=None,
+                    task_output=None,
                     exception=e,
                 )
             }

fractal_server/runner/v2/runner_functions.py

@@ -1,7 +1,7 @@
-from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 from typing import Literal
+from typing import Protocol
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -20,7 +20,12 @@ from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import TaskType
 from fractal_server.exceptions import UnreachableBranchError
 from fractal_server.logger import set_logger
+from fractal_server.runner.config import JobRunnerConfigLocal
+from fractal_server.runner.config import JobRunnerConfigSLURM
 from fractal_server.runner.executors.base_runner import BaseRunner
+from fractal_server.runner.executors.slurm_common.slurm_config import (
+    SlurmConfig,
+)
 from fractal_server.runner.task_files import enrich_task_files_multisubmit
 from fractal_server.runner.task_files import SUBMIT_PREFIX
 from fractal_server.runner.task_files import TaskFiles
@@ -35,6 +40,32 @@ from fractal_server.runner.v2.task_interface import (
     _cast_and_validate_TaskOutput,
 )
 
+
+class GetRunnerConfigTypeLocal(Protocol):
+    def __call__(
+        self,
+        shared_config: JobRunnerConfigLocal,
+        wftask: WorkflowTaskV2,
+        which_type: Literal["non_parallel", "parallel"],
+        tot_tasks: int,
+    ) -> JobRunnerConfigLocal:
+        ...
+
+
+class GetRunnerConfigTypeSLURM(Protocol):
+    def __call__(
+        self,
+        shared_config: JobRunnerConfigSLURM,
+        wftask: WorkflowTaskV2,
+        which_type: Literal["non_parallel", "parallel"],
+        tot_tasks: int,
+    ) -> SlurmConfig:
+        ...
+
+
+GetRunnerConfigType = GetRunnerConfigTypeLocal | GetRunnerConfigTypeSLURM
+
+
 __all__ = [
     "run_v2_task_parallel",
     "run_v2_task_non_parallel",
@@ -126,15 +157,7 @@ def run_v2_task_non_parallel(
     workflow_dir_local: Path,
     workflow_dir_remote: Path,
     runner: BaseRunner,
-    get_runner_config: Callable[
-        [
-            WorkflowTaskV2,
-            Literal["non_parallel", "parallel"],
-            Path | None,
-            int,
-        ],
-        Any,
-    ],
+    get_runner_config: GetRunnerConfigType,
     dataset_id: int,
     history_run_id: int,
     task_type: Literal[TaskType.NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL],
@@ -163,8 +186,10 @@ def run_v2_task_non_parallel(
     )
 
     runner_config = get_runner_config(
+        shared_config=runner.shared_config,
         wftask=wftask,
         which_type="non_parallel",
+        tot_tasks=1,
     )
 
     function_kwargs = {
@@ -251,15 +276,7 @@ def run_v2_task_parallel(
     runner: BaseRunner,
     workflow_dir_local: Path,
     workflow_dir_remote: Path,
-    get_runner_config: Callable[
-        [
-            WorkflowTaskV2,
-            Literal["non_parallel", "parallel"],
-            Path | None,
-            int,
-        ],
-        Any,
-    ],
+    get_runner_config: GetRunnerConfigType,
     dataset_id: int,
     history_run_id: int,
     user_id: int,
@@ -278,6 +295,7 @@ def run_v2_task_parallel(
     )
 
     runner_config = get_runner_config(
+        shared_config=runner.shared_config,
         wftask=wftask,
         which_type="parallel",
         tot_tasks=len(images),
@@ -382,15 +400,7 @@ def run_v2_task_compound(
     runner: BaseRunner,
     workflow_dir_local: Path,
     workflow_dir_remote: Path,
-    get_runner_config: Callable[
-        [
-            WorkflowTaskV2,
-            Literal["non_parallel", "parallel"],
-            Path | None,
-            int,
-        ],
-        Any,
-    ],
+    get_runner_config: GetRunnerConfigType,
     dataset_id: int,
     history_run_id: int,
     task_type: Literal[TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND],
@@ -407,9 +417,12 @@ def run_v2_task_compound(
     )
 
     runner_config_init = get_runner_config(
+        shared_config=runner.shared_config,
         wftask=wftask,
         which_type="non_parallel",
+        tot_tasks=1,
     )
+
     # 3/A: non-parallel init task
     function_kwargs = {
         "zarr_dir": zarr_dir,
@@ -505,6 +518,7 @@ def run_v2_task_compound(
         return init_outcome, num_tasks
 
     runner_config_compute = get_runner_config(
+        shared_config=runner.shared_config,
         wftask=wftask,
         which_type="parallel",
         tot_tasks=len(parallelization_list),