fractal-server 2.16.5__py3-none-any.whl → 2.17.0__py3-none-any.whl

fractal_server/runner/v2/submit_workflow.py

@@ -8,35 +8,53 @@ the individual backends.
 import os
 import traceback
 from pathlib import Path
+from typing import Protocol
 
 from sqlalchemy.orm import Session as DBSyncSession
 
-from ...config import get_settings
-from ...logger import get_logger
-from ...logger import reset_logger_handlers
-from ...logger import set_logger
-from ...ssh._fabric import FractalSSH
-from ...syringe import Inject
-from ...utils import get_timestamp
-from ...zip_tools import _zip_folder_to_file_and_remove
-from ..exceptions import JobExecutionError
-from ..filenames import WORKFLOW_LOG_FILENAME
 from ._local import process_workflow as local_process_workflow
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
 from fractal_server import __VERSION__
 from fractal_server.app.db import DB
-from fractal_server.app.models import UserSettings
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import JobV2
+from fractal_server.app.models.v2 import Profile
+from fractal_server.app.models.v2 import Resource
 from fractal_server.app.models.v2 import WorkflowV2
 from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.app.schemas.v2 import ResourceType
+from fractal_server.logger import get_logger
+from fractal_server.logger import reset_logger_handlers
+from fractal_server.logger import set_logger
+from fractal_server.runner.exceptions import JobExecutionError
+from fractal_server.runner.filenames import WORKFLOW_LOG_FILENAME
+from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.types import AttributeFilters
+from fractal_server.utils import get_timestamp
+from fractal_server.zip_tools import _zip_folder_to_file_and_remove
 
 
-_backends = {}
-_backends["local"] = local_process_workflow
-_backends["slurm"] = slurm_sudo_process_workflow
-_backends["slurm_ssh"] = slurm_ssh_process_workflow
+class ProcessWorkflowType(Protocol):
+    def __call__(
+        self,
+        *,
+        workflow: WorkflowV2,
+        dataset: DatasetV2,
+        workflow_dir_local: Path,
+        job_id: int,
+        workflow_dir_remote: Path | None,
+        first_task_index: int | None,
+        last_task_index: int | None,
+        logger_name: str,
+        job_attribute_filters: AttributeFilters,
+        job_type_filters: dict[str, bool],
+        user_id: int,
+        resource: Resource,
+        profile: Profile,
+        user_cache_dir: str,
+    ) -> None:
+        ...
 
 
 def fail_job(
@@ -67,10 +85,10 @@ def submit_workflow(
     dataset_id: int,
     job_id: int,
     user_id: int,
-    user_settings: UserSettings,
+    user_cache_dir: str,
+    resource: Resource,
+    profile: Profile,
     worker_init: str | None = None,
-    slurm_user: str | None = None,
-    user_cache_dir: str | None = None,
     fractal_ssh: FractalSSH | None = None,
 ) -> None:
     """
@@ -93,16 +111,18 @@ def submit_workflow(
             Custom executor parameters that get parsed before the execution of
             each task.
         user_cache_dir:
-            Cache directory (namely a path where the user can write); for the
-            slurm backend, this is used as a base directory for
-            `job.working_dir_user`.
-        slurm_user:
-            The username to impersonate for the workflow execution, for the
-            slurm backend.
+            Cache directory (namely a path where the user can write). For
+            `slurm_sudo` backend, this is both a base directory for
+            `job.working_dir_user`. For `slurm_sudo` and `slurm_ssh` backends,
+            this is used for `user_local_exports`.
+        resource:
+            Computational resource to be used for this job (e.g. a SLURM
+            cluster).
+        profile:
+           Computational profile to be used for this job.
+        fractal_ssh: SSH object, for when `resource.type = "slurm_ssh"`.
     """
     # Declare runner backend and set `process_workflow` function
-    settings = Inject(get_settings)
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     logger_name = f"WF{workflow_id}_job{job_id}"
     logger = set_logger(logger_name=logger_name)
 
@@ -135,66 +155,30 @@ def submit_workflow(
             )
             return
 
-        # Declare runner backend and set `process_workflow` function
-        settings = Inject(get_settings)
-        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
         try:
-            process_workflow = _backends[settings.FRACTAL_RUNNER_BACKEND]
-        except KeyError as e:
-            fail_job(
-                db=db_sync,
-                job=job,
-                log_msg=(
-                    f"Invalid {FRACTAL_RUNNER_BACKEND=}.\n"
-                    f"Original KeyError: {str(e)}"
-                ),
-                logger_name=logger_name,
-                emit_log=True,
-            )
-            return
-
-        # Define and create server-side working folder
-        WORKFLOW_DIR_LOCAL = Path(job.working_dir)
-        if WORKFLOW_DIR_LOCAL.exists():
-            fail_job(
-                db=db_sync,
-                job=job,
-                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
-                logger_name=logger_name,
-                emit_log=True,
-            )
-            return
+            # Define local/remote folders, and create local folder
+            local_job_dir = Path(job.working_dir)
+            remote_job_dir = Path(job.working_dir_user)
+            match resource.type:
+                case ResourceType.LOCAL:
+                    local_job_dir.mkdir(parents=True, exist_ok=False)
+                case ResourceType.SLURM_SUDO:
+                    original_umask = os.umask(0)
+                    local_job_dir.mkdir(
+                        parents=True, mode=0o755, exist_ok=False
+                    )
+                    os.umask(original_umask)
+                case ResourceType.SLURM_SSH:
+                    local_job_dir.mkdir(parents=True, exist_ok=False)
 
-        try:
-            # Create WORKFLOW_DIR_LOCAL and define WORKFLOW_DIR_REMOTE
-            if FRACTAL_RUNNER_BACKEND == "local":
-                WORKFLOW_DIR_LOCAL.mkdir(parents=True)
-                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-            elif FRACTAL_RUNNER_BACKEND == "slurm":
-                original_umask = os.umask(0)
-                WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
-                os.umask(original_umask)
-                WORKFLOW_DIR_REMOTE = (
-                    Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
-                )
-            elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-                WORKFLOW_DIR_LOCAL.mkdir(parents=True)
-                WORKFLOW_DIR_REMOTE = (
-                    Path(user_settings.ssh_jobs_dir) / WORKFLOW_DIR_LOCAL.name
-                )
-            else:
-                raise ValueError(
-                    "Invalid FRACTAL_RUNNER_BACKEND="
-                    f"{settings.FRACTAL_RUNNER_BACKEND}."
-                )
         except Exception as e:
             error_type = type(e).__name__
             fail_job(
                 db=db_sync,
                 job=job,
                 log_msg=(
-                    f"{error_type} error occurred while creating job folder "
-                    f"and subfolders.\nOriginal error: {str(e)}"
+                    f"{error_type} error while creating local job folder."
+                    f" Original error: {str(e)}"
                 ),
                 logger_name=logger_name,
                 emit_log=True,
@@ -218,7 +202,7 @@ def submit_workflow(
             db_sync.refresh(wftask)
 
         # Write logs
-        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
+        log_file_path = local_job_dir / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
             log_file_path=log_file_path,
@@ -228,14 +212,10 @@ def submit_workflow(
             f"more logs at {str(log_file_path)}"
         )
         logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
-        logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
-        if FRACTAL_RUNNER_BACKEND == "slurm":
-            logger.debug(f"slurm_user: {slurm_user}")
-            logger.debug(f"slurm_account: {job.slurm_account}")
-            logger.debug(f"worker_init: {worker_init}")
-        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-            logger.debug(f"ssh_user: {user_settings.ssh_username}")
-            logger.debug(f"base dir: {user_settings.ssh_tasks_dir}")
+        logger.debug(f"Resource name: {resource.name}")
+        logger.debug(f"Profile name: {profile.name}")
+        logger.debug(f"Username: {profile.username}")
+        if resource.type in [ResourceType.SLURM_SUDO, ResourceType.SLURM_SSH]:
             logger.debug(f"slurm_account: {job.slurm_account}")
             logger.debug(f"worker_init: {worker_init}")
         logger.debug(f"job.id: {job.id}")
@@ -247,40 +227,39 @@ def submit_workflow(
         job_working_dir = job.working_dir
 
     try:
-        if FRACTAL_RUNNER_BACKEND == "local":
-            process_workflow = local_process_workflow
-            backend_specific_kwargs = {}
-        elif FRACTAL_RUNNER_BACKEND == "slurm":
-            process_workflow = slurm_sudo_process_workflow
-            backend_specific_kwargs = dict(
-                slurm_user=slurm_user,
-                slurm_account=job.slurm_account,
-                user_cache_dir=user_cache_dir,
-            )
-        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-            process_workflow = slurm_ssh_process_workflow
-            backend_specific_kwargs = dict(
-                fractal_ssh=fractal_ssh,
-                slurm_account=job.slurm_account,
-            )
-        else:
-            raise RuntimeError(
-                f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}"
-            )
+        process_workflow: ProcessWorkflowType
+        match resource.type:
+            case ResourceType.LOCAL:
+                process_workflow = local_process_workflow
+                backend_specific_kwargs = {}
+            case ResourceType.SLURM_SUDO:
+                process_workflow = slurm_sudo_process_workflow
+                backend_specific_kwargs = dict(
+                    slurm_account=job.slurm_account,
+                )
+            case ResourceType.SLURM_SSH:
+                process_workflow = slurm_ssh_process_workflow
+                backend_specific_kwargs = dict(
+                    fractal_ssh=fractal_ssh,
+                    slurm_account=job.slurm_account,
+                )
 
         process_workflow(
             workflow=workflow,
             dataset=dataset,
             job_id=job_id,
             user_id=user_id,
-            workflow_dir_local=WORKFLOW_DIR_LOCAL,
-            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
+            workflow_dir_local=local_job_dir,
+            workflow_dir_remote=remote_job_dir,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
             job_attribute_filters=job.attribute_filters,
             job_type_filters=job.type_filters,
+            resource=resource,
+            profile=profile,
+            user_cache_dir=user_cache_dir,
             **backend_specific_kwargs,
         )
 

fractal_server/runner/versions.py

@@ -1,18 +1,23 @@
 import json
 import sys
+from typing import TypedDict
 
 import fractal_server
 
 
-def get_versions() -> dict[str, list[int] | str]:
+class VersionsType(TypedDict):
+    python: tuple[int, ...]
+    fractal_server: str
+
+
+def get_versions() -> VersionsType:
     """
     Extract versions of Python and fractal-server.
     """
-    versions = dict(
+    return dict(
         python=tuple(sys.version_info[:3]),
         fractal_server=fractal_server.__VERSION__,
     )
-    return versions
 
 
 if __name__ == "__main__":

fractal_server/ssh/_fabric.py

@@ -84,7 +84,7 @@ def _acquire_lock_with_timeout(
     """
     Given a `threading.Lock` object, try to acquire it within a given timeout.
 
-    Arguments:
+    Args:
         lock:
         label:
         timeout:
@@ -177,7 +177,7 @@ class FractalSSH:
         """
         Log and re-raise an exception from a FractalSSH method.
 
-        Arguments:
+        Args:
             message: Additional message to be logged.
             e: Original exception
         """
@@ -686,7 +686,7 @@ class FractalSSHList:
 
         Note: Changing `_data` requires acquiring `_lock`.
 
-        Arguments:
+        Args:
             host:
             user:
             key_path:
@@ -732,7 +732,7 @@ class FractalSSHList:
         """
         Return whether a given key is present in the collection.
 
-        Arguments:
+        Args:
             host:
             user:
             key_path:
@@ -752,7 +752,7 @@ class FractalSSHList:
 
         Note: Changing `_data` requires acquiring `_lock`.
 
-        Arguments:
+        Args:
             host:
             user:
             key_path:
@@ -780,7 +780,7 @@ class FractalSSHList:
         """
         Close all `FractalSSH` objects in the collection.
 
-        Arguments:
+        Args:
             timeout:
                 Timeout for `FractalSSH._lock` acquisition, to be obtained
                 before closing.

fractal_server/tasks/config/__init__.py

@@ -0,0 +1,3 @@
+from ._pixi import PixiSLURMConfig  # noqa F401
+from ._pixi import TasksPixiSettings  # noqa F401
+from ._python import TasksPythonSettings  # noqa F401

fractal_server/tasks/config/_pixi.py

@@ -0,0 +1,127 @@
+from pathlib import Path
+from typing import Annotated
+
+from pydantic import AfterValidator
+from pydantic import BaseModel
+from pydantic import model_validator
+from pydantic import PositiveInt
+
+from fractal_server.types import DictStrStr
+from fractal_server.types import NonEmptyStr
+
+
+def _check_pixi_slurm_memory(mem: str) -> str:
+    if mem[-1] not in ["K", "M", "G", "T"]:
+        raise ValueError(
+            f"Invalid memory requirement {mem=} for `pixi`, "
+            "please set a K/M/G/T units suffix."
+        )
+    return mem
+
+
+class PixiSLURMConfig(BaseModel):
+    """
+    Parameters that are passed directly to a `sbatch` command.
+
+    See https://slurm.schedmd.com/sbatch.html.
+    """
+
+    partition: NonEmptyStr
+    """
+    `-p, --partition=<partition_names>`
+    """
+    cpus: PositiveInt
+    """
+    `-c, --cpus-per-task=<ncpus>
+    """
+    mem: Annotated[NonEmptyStr, AfterValidator(_check_pixi_slurm_memory)]
+    """
+    `--mem=<size>[units]` (examples: `"10M"`, `"10G"`).
+    From `sbatch` docs: Specify the real memory required per node. Default
+    units are megabytes. Different units can be specified using the suffix
+    [K|M|G|T].
+    """
+    time: NonEmptyStr
+    """
+    `-t, --time=<time>`.
+    From `sbatch` docs: "A time limit of zero requests that no time limit be
+    imposed. Acceptable time formats include "minutes", "minutes:seconds",
+    "hours:minutes:seconds", "days-hours", "days-hours:minutes" and
+    "days-hours:minutes:seconds".
+    """
+
+
+class TasksPixiSettings(BaseModel):
+    """
+    Configuration for `pixi` Task collection.
+    """
+
+    versions: DictStrStr
+    """
+    Dictionary mapping `pixi` versions (e.g. `0.47.0`) to the corresponding
+    folders (e.g. `/somewhere/pixi/0.47.0` - if the binary is
+    `/somewhere/pixi/0.47.0/bin/pixi`).
+    """
+    default_version: str
+    """
+    Default task-collection `pixi` version.
+    """
+    PIXI_CONCURRENT_SOLVES: int = 4
+    """
+    Value of
+    [`--concurrent-solves`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-solves)
+    for `pixi install`.
+    """
+    PIXI_CONCURRENT_DOWNLOADS: int = 4
+    """
+    Value of
+    [`--concurrent-downloads`](https://pixi.sh/latest/reference/cli/pixi/install/#arg---concurrent-downloads)
+    for `pixi install`.
+    """
+    TOKIO_WORKER_THREADS: int = 2
+    """
+    From
+    [Tokio documentation](
+    https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
+    )
+    :
+
+        The core threads are where all asynchronous code runs,
+        and Tokio will by default spawn one for each CPU core.
+        You can use the environment variable `TOKIO_WORKER_THREADS` to override
+        the default value.
+    """
+    DEFAULT_ENVIRONMENT: str = "default"
+    """
+    Default pixi environment name.
+    """
+    DEFAULT_PLATFORM: str = "linux-64"
+    """
+    Default platform for pixi.
+    """
+    SLURM_CONFIG: PixiSLURMConfig | None = None
+    """
+    Required when using `pixi` in a SSH/SLURM deployment.
+    """
+
+    @model_validator(mode="after")
+    def check_pixi_settings(self):
+        if self.default_version not in self.versions:
+            raise ValueError(
+                f"Default version '{self.default_version}' not in "
+                f"available version {list(self.versions.keys())}."
+            )
+
+        pixi_base_dir = Path(self.versions[self.default_version]).parent
+
+        for key, value in self.versions.items():
+            pixi_path = Path(value)
+
+            if pixi_path.parent != pixi_base_dir:
+                raise ValueError(
+                    f"{pixi_path=} is not located within the {pixi_base_dir=}."
+                )
+            if pixi_path.name != key:
+                raise ValueError(f"{pixi_path.name=} is not equal to {key=}")
+
+        return self

fractal_server/tasks/config/_python.py

@@ -0,0 +1,51 @@
+from typing import Literal
+from typing import Self
+
+from pydantic import BaseModel
+from pydantic import model_validator
+
+from fractal_server.types import AbsolutePathStr
+from fractal_server.types import NonEmptyStr
+
+
+class TasksPythonSettings(BaseModel):
+    """
+    Configuration for the Python base interpreters to be used for task venvs.
+
+    For task collection to work, there must be one or more base Python
+    interpreters available on your system.
+    """
+
+    default_version: NonEmptyStr
+    """
+    Default task-collection Python version (must be a key of `versions`).
+    """
+    versions: dict[
+        Literal[
+            "3.9",
+            "3.10",
+            "3.11",
+            "3.12",
+            "3.13",
+            "3.14",
+        ],
+        AbsolutePathStr,
+    ]
+    """
+    Dictionary mapping Python versions to the corresponding interpreters.
+    """
+
+    pip_cache_dir: AbsolutePathStr | None = None
+    """
+    Argument for `--cache-dir` option of `pip install`, if set.
+    """
+
+    @model_validator(mode="after")
+    def _validate_versions(self) -> Self:
+        if self.default_version not in self.versions.keys():
+            raise ValueError(
+                f"The default Python version ('{self.default_version}') is "
+                f"not available in {list(self.versions.keys())}."
+            )
+
+        return self

fractal_server/tasks/v2/local/_utils.py

@@ -1,15 +1,13 @@
 from pathlib import Path
 
 from ..utils_pixi import simplify_pyproject_toml
+from fractal_server.app.models import Resource
 from fractal_server.app.schemas.v2 import TaskCreateV2
-from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
-from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.utils_templates import customize_template
 from fractal_server.utils import execute_command_sync
 
-
 logger = set_logger(__name__)
 
 
@@ -78,7 +76,10 @@ def check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
                 )
 
 
-def edit_pyproject_toml_in_place_local(pyproject_toml_path: Path) -> None:
+def edit_pyproject_toml_in_place_local(
+    pyproject_toml_path: Path,
+    resource: Resource,
+) -> None:
     """
     Wrapper of `simplify_pyproject_toml`, with I/O.
     """
@@ -88,11 +89,10 @@ def edit_pyproject_toml_in_place_local(pyproject_toml_path: Path) -> None:
         pyproject_contents = f.read()
 
     # Simplify contents
-    settings = Inject(get_settings)
     new_pyproject_contents = simplify_pyproject_toml(
         original_toml_string=pyproject_contents,
-        pixi_environment=settings.pixi.DEFAULT_ENVIRONMENT,
-        pixi_platform=settings.pixi.DEFAULT_PLATFORM,
+        pixi_environment=resource.tasks_pixi_config["DEFAULT_ENVIRONMENT"],
+        pixi_platform=resource.tasks_pixi_config["DEFAULT_PLATFORM"],
     )
     # Write new `pyproject.toml`
     with pyproject_toml_path.open("w") as f:

fractal_server/tasks/v2/local/collect.py

@@ -7,6 +7,8 @@ from tempfile import TemporaryDirectory
 from ..utils_database import create_db_tasks_and_update_task_group_sync
 from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import Profile
+from fractal_server.app.models import Resource
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.schemas.v2 import FractalUploadedFile
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
@@ -25,7 +27,7 @@ from fractal_server.tasks.v2.utils_background import get_current_log
 from fractal_server.tasks.v2.utils_background import prepare_tasks_metadata
 from fractal_server.tasks.v2.utils_package_names import compare_package_names
 from fractal_server.tasks.v2.utils_python_interpreter import (
-    get_python_interpreter_v2,
+    get_python_interpreter,
 )
 from fractal_server.tasks.v2.utils_templates import get_collection_replacements
 from fractal_server.tasks.v2.utils_templates import (
@@ -39,6 +41,8 @@ def collect_local(
     *,
     task_group_activity_id: int,
     task_group_id: int,
+    resource: Resource,
+    profile: Profile,
     wheel_file: FractalUploadedFile | None = None,
 ) -> None:
     """
@@ -52,9 +56,10 @@ def collect_local(
     https://github.com/encode/starlette/blob/master/starlette/background.py).
 
 
-    Arguments:
+    Args:
         task_group_id:
         task_group_activity_id:
+        resource: Resource
         wheel_file:
     """
 
@@ -111,11 +116,14 @@ def collect_local(
                     task_group = add_commit_refresh(obj=task_group, db=db)
 
                 # Prepare replacements for templates
+                python_bin = get_python_interpreter(
+                    python_version=task_group.python_version,
+                    resource=resource,
+                )
                 replacements = get_collection_replacements(
                     task_group=task_group,
-                    python_bin=get_python_interpreter_v2(
-                        python_version=task_group.python_version
-                    ),
+                    python_bin=python_bin,
+                    resource=resource,
                 )
 
                 # Prepare common arguments for `_customize_and_run_template``