fractal-server 2.14.4a0__py3-none-any.whl → 2.14.6__py3-none-any.whl

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -5,7 +5,6 @@ import subprocess  # nosec
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import Optional
 
 from ..slurm_common.base_slurm_runner import BaseSlurmRunner
 from ..slurm_common.slurm_job_task_models import SlurmJob
@@ -21,7 +20,7 @@ logger = set_logger(__name__)
 
 def _subprocess_run_or_raise(
     full_command: str,
-) -> Optional[subprocess.CompletedProcess]:
+) -> subprocess.CompletedProcess | None:
     try:
         output = subprocess.run(  # nosec
             shlex.split(full_command),
@@ -43,7 +42,7 @@ def _subprocess_run_or_raise(
 
 class SudoSlurmRunner(BaseSlurmRunner):
     slurm_user: str
-    slurm_account: Optional[str] = None
+    slurm_account: str | None = None
 
     def __init__(
         self,
@@ -51,11 +50,11 @@ class SudoSlurmRunner(BaseSlurmRunner):
         # Common
         root_dir_local: Path,
         root_dir_remote: Path,
-        common_script_lines: Optional[list[str]] = None,
-        user_cache_dir: Optional[str] = None,
-        poll_interval: Optional[int] = None,
+        common_script_lines: list[str] | None = None,
+        user_cache_dir: str | None = None,
+        poll_interval: int | None = None,
         # Specific
-        slurm_account: Optional[str] = None,
+        slurm_account: str | None = None,
         slurm_user: str,
     ) -> None:
         """
@@ -102,17 +101,13 @@ class SudoSlurmRunner(BaseSlurmRunner):
             source_target_list.extend(
                 [
                     (
-                        task.output_pickle_file_remote,
-                        task.output_pickle_file_local,
+                        task.output_file_remote,
+                        task.output_file_local,
                     ),
                     (
                         task.task_files.log_file_remote,
                         task.task_files.log_file_local,
                     ),
-                    (
-                        task.task_files.args_file_remote,
-                        task.task_files.args_file_local,
-                    ),
                     (
                         task.task_files.metadiff_file_remote,
                         task.task_files.metadiff_file_local,
@@ -121,17 +116,14 @@ class SudoSlurmRunner(BaseSlurmRunner):
             )
 
         for source, target in source_target_list:
-            # NOTE: By setting encoding=None, we read/write bytes instead
-            # of strings; this is needed to also handle pickle files.
             try:
                 res = _run_command_as_user(
                     cmd=f"cat {source}",
                     user=self.slurm_user,
-                    encoding=None,
                     check=True,
                 )
                 # Write local file
-                with open(target, "wb") as f:
+                with open(target, "w") as f:
                     f.write(res.stdout)
                 logger.debug(
                     f"[_fetch_artifacts_single_job] Copied {source} into "
@@ -171,7 +163,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
         res = _run_command_as_user(
             cmd=cmd,
             user=self.slurm_user,
-            encoding="utf-8",
             check=True,
         )
         return res.stdout

fractal_server/app/runner/set_start_and_last_task_index.py

@@ -1,10 +1,7 @@
-from typing import Optional
-
-
 def set_start_and_last_task_index(
     num_tasks: int,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
+    first_task_index: int | None = None,
+    last_task_index: int | None = None,
 ) -> tuple[int, int]:
     """
     Handle `first_task_index` and `last_task_index`, by setting defaults and

fractal_server/app/runner/shutdown.py

@@ -38,21 +38,15 @@ async def cleanup_after_shutdown(*, jobsV2: list[int], logger_name: str):
 
             if len(jobsV2_db) == 0:
                 logger.info(
-                    (
-                        "All jobs associated to this app are "
-                        "either done or failed. Exit."
-                    )
+                    "All jobs associated to this app are "
+                    "either done or failed. Exit."
                 )
                 return
             else:
-                logger.info(
-                    (f"Some jobs are still 'submitted' " f"{jobsV2_db=}")
-                )
+                logger.info(f"Some jobs are still 'submitted' {jobsV2_db=}")
         logger.info(
-            (
-                "Graceful shutdown reached its maximum time, "
-                "but some jobs are still submitted"
-            )
+            "Graceful shutdown reached its maximum time, "
+            "but some jobs are still submitted"
         )
 
         for job in jobsV2_db:

fractal_server/app/runner/task_files.py

@@ -1,6 +1,4 @@
 from pathlib import Path
-from typing import Optional
-from typing import Union
 
 from pydantic import BaseModel
 
@@ -12,7 +10,7 @@ MULTISUBMIT_PREFIX = "par"
 
 
 def task_subfolder_name(
-    order: Union[int, str],
+    order: int | str,
     task_name: str,
 ) -> str:
     """
@@ -48,8 +46,8 @@ class TaskFiles(BaseModel):
     task_order: int
 
     # Per-single-component
-    component: Optional[str] = None
-    prefix: Optional[str] = None
+    component: str | None = None
+    prefix: str | None = None
 
     def _check_component(self):
         if self.component is None:
@@ -134,14 +132,6 @@ class TaskFiles(BaseModel):
     def metadiff_file_remote(self) -> str:
         return self.metadiff_file_remote_path.as_posix()
 
-    @property
-    def remote_files_dict(self) -> dict[str, str]:
-        return dict(
-            args_file_remote=self.args_file_remote,
-            metadiff_file_remote=self.metadiff_file_remote,
-            log_file_remote=self.log_file_remote,
-        )
-
 
 def enrich_task_files_multisubmit(
     *,

fractal_server/app/runner/v2/_local.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from typing import Optional
 
 from ...models.v2 import DatasetV2
 from ...models.v2 import WorkflowV2
@@ -16,9 +15,9 @@ def process_workflow(
     dataset: DatasetV2,
     workflow_dir_local: Path,
     job_id: int,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
+    workflow_dir_remote: Path | None = None,
+    first_task_index: int | None = None,
+    last_task_index: int | None = None,
     logger_name: str,
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],

fractal_server/app/runner/v2/_slurm_ssh.py

@@ -13,11 +13,9 @@
 """
 Slurm Backend
 
-This backend runs fractal workflows in a SLURM cluster using Clusterfutures
-Executor objects.
+This backend runs fractal workflows in a SLURM cluster.
 """
 from pathlib import Path
-from typing import Optional
 
 from ....ssh._fabric import FractalSSH
 from ...models.v2 import DatasetV2
@@ -39,14 +37,14 @@ def process_workflow(
     dataset: DatasetV2,
     workflow_dir_local: Path,
     job_id: int,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
+    workflow_dir_remote: Path | None = None,
+    first_task_index: int | None = None,
+    last_task_index: int | None = None,
     logger_name: str,
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     fractal_ssh: FractalSSH,
-    worker_init: Optional[str] = None,
+    worker_init: str | None = None,
     user_id: int,
     **kwargs,  # not used
 ) -> None:

fractal_server/app/runner/v2/_slurm_sudo.py

@@ -13,11 +13,9 @@
 """
 Slurm Backend
 
-This backend runs fractal workflows in a SLURM cluster using Clusterfutures
-Executor objects.
+This backend runs fractal workflows in a SLURM cluster.
 """
 from pathlib import Path
-from typing import Optional
 
 from ...models.v2 import DatasetV2
 from ...models.v2 import WorkflowV2
@@ -34,18 +32,18 @@ def process_workflow(
     dataset: DatasetV2,
     workflow_dir_local: Path,
     job_id: int,
-    workflow_dir_remote: Optional[Path] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
+    workflow_dir_remote: Path | None = None,
+    first_task_index: int | None = None,
+    last_task_index: int | None = None,
     logger_name: str,
     job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
     # Slurm-specific
-    user_cache_dir: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
-    worker_init: Optional[str] = None,
+    user_cache_dir: str | None = None,
+    slurm_user: str | None = None,
+    slurm_account: str | None = None,
+    worker_init: str | None = None,
 ) -> None:
     """
     Process workflow (SLURM backend public interface).

fractal_server/app/runner/v2/runner.py

@@ -1,11 +1,10 @@
 import logging
+from collections.abc import Callable
 from copy import copy
 from copy import deepcopy
 from pathlib import Path
 from typing import Any
-from typing import Callable
 from typing import Literal
-from typing import Optional
 
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import delete
@@ -76,13 +75,13 @@ def execute_tasks_v2(
     user_id: int,
     workflow_dir_local: Path,
     job_id: int,
-    workflow_dir_remote: Optional[Path] = None,
-    logger_name: Optional[str] = None,
+    workflow_dir_remote: Path | None = None,
+    logger_name: str | None = None,
     get_runner_config: Callable[
         [
             WorkflowTaskV2,
             Literal["non_parallel", "parallel"],
-            Optional[Path],
+            Path | None,
         ],
         Any,
     ],

fractal_server/app/runner/v2/runner_functions.py

@@ -1,9 +1,7 @@
-import functools
+from collections.abc import Callable
 from pathlib import Path
 from typing import Any
-from typing import Callable
 from typing import Literal
-from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -12,7 +10,6 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskOutputValidationError
 from .db_tools import update_status_of_history_unit
 from .deduplicate_list import deduplicate_list
-from .runner_functions_low_level import run_single_task
 from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
@@ -131,7 +128,7 @@ def run_v2_task_non_parallel(
         [
             WorkflowTaskV2,
             Literal["non_parallel", "parallel"],
-            Optional[Path],
+            Path | None,
             int,
         ],
         Any,
@@ -207,13 +204,10 @@ def run_v2_task_non_parallel(
         )
 
     result, exception = runner.submit(
-        functools.partial(
-            run_single_task,
-            command=task.command_non_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_non_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         parameters=function_kwargs,
         task_type=task_type,
         task_files=task_files,
@@ -256,7 +250,7 @@ def run_v2_task_parallel(
         [
             WorkflowTaskV2,
             Literal["non_parallel", "parallel"],
-            Optional[Path],
+            Path | None,
             int,
         ],
         Any,
@@ -335,13 +329,10 @@ def run_v2_task_parallel(
         )
 
     results, exceptions = runner.multisubmit(
-        functools.partial(
-            run_single_task,
-            command=task.command_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         list_parameters=list_function_kwargs,
         task_type="parallel",
         list_task_files=list_task_files,
@@ -390,7 +381,7 @@ def run_v2_task_compound(
         [
             WorkflowTaskV2,
             Literal["non_parallel", "parallel"],
-            Optional[Path],
+            Path | None,
             int,
         ],
         Any,
@@ -456,13 +447,10 @@ def run_v2_task_compound(
             ],
         )
     result, exception = runner.submit(
-        functools.partial(
-            run_single_task,
-            command=task.command_non_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_non_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         parameters=function_kwargs,
         task_type=task_type,
         task_files=task_files_init,
@@ -559,13 +547,10 @@ def run_v2_task_compound(
         history_unit_ids = [history_unit.id for history_unit in history_units]
 
     results, exceptions = runner.multisubmit(
-        functools.partial(
-            run_single_task,
-            command=task.command_parallel,
-            workflow_task_order=wftask.order,
-            workflow_task_id=wftask.task_id,
-            task_name=wftask.task.name,
-        ),
+        base_command=task.command_parallel,
+        workflow_task_order=wftask.order,
+        workflow_task_id=wftask.task_id,
+        task_name=wftask.task.name,
         list_parameters=list_function_kwargs,
         task_type=task_type,
         list_task_files=list_task_files,

fractal_server/app/runner/v2/submit_workflow.py

@@ -8,7 +8,6 @@ the individual backends.
 import os
 import traceback
 from pathlib import Path
-from typing import Optional
 
 from sqlalchemy.orm import Session as DBSyncSession
 
@@ -69,10 +68,10 @@ def submit_workflow(
     job_id: int,
     user_id: int,
     user_settings: UserSettings,
-    worker_init: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    user_cache_dir: Optional[str] = None,
-    fractal_ssh: Optional[FractalSSH] = None,
+    worker_init: str | None = None,
+    slurm_user: str | None = None,
+    user_cache_dir: str | None = None,
+    fractal_ssh: FractalSSH | None = None,
 ) -> None:
     """
     Prepares a workflow and applies it to a dataset
@@ -109,11 +108,9 @@ def submit_workflow(
 
     with next(DB.get_sync_db()) as db_sync:
         try:
-            job: Optional[JobV2] = db_sync.get(JobV2, job_id)
-            dataset: Optional[DatasetV2] = db_sync.get(DatasetV2, dataset_id)
-            workflow: Optional[WorkflowV2] = db_sync.get(
-                WorkflowV2, workflow_id
-            )
+            job: JobV2 | None = db_sync.get(JobV2, job_id)
+            dataset: DatasetV2 | None = db_sync.get(DatasetV2, dataset_id)
+            workflow: WorkflowV2 | None = db_sync.get(WorkflowV2, workflow_id)
         except Exception as e:
             logger.error(
                 f"Error connecting to the database. Original error: {str(e)}"

fractal_server/app/runner/v2/task_interface.py

@@ -1,5 +1,4 @@
 from typing import Any
-from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -57,7 +56,7 @@ class InitTaskOutput(BaseModel):
 
 def _cast_and_validate_TaskOutput(
     task_output: dict[str, Any]
-) -> Optional[TaskOutput]:
+) -> TaskOutput | None:
     try:
         validated_task_output = TaskOutput(**task_output)
         return validated_task_output
@@ -71,7 +70,7 @@ def _cast_and_validate_TaskOutput(
 
 def _cast_and_validate_InitTaskOutput(
     init_task_output: dict[str, Any],
-) -> Optional[InitTaskOutput]:
+) -> InitTaskOutput | None:
     try:
         validated_init_task_output = InitTaskOutput(**init_task_output)
         return validated_init_task_output

fractal_server/app/runner/versions.py

@@ -1,25 +1,15 @@
 import json
 import sys
-from typing import Union
-
-import cloudpickle
 
 import fractal_server
 
 
-def get_versions() -> dict[str, Union[list[int], str]]:
+def get_versions() -> dict[str, list[int] | str]:
     """
-    Extract versions of Python, fractal-server and cloudpickle.
-
-    This information is useful to check compatibility of two Python
-    interpreters when running tasks: the current interpreter (which prepares
-    the input pickles and orchestrates workflow execution) and a remote
-    interpreter (e.g. the one defined in the `FRACTAL_SLURM_WORKER_PYTHON`
-    configuration variable) that executes the tasks.
+    Extract versions of Python and fractal-server.
     """
     versions = dict(
-        python=list(sys.version_info[:3]),
-        cloudpickle=cloudpickle.__version__,
+        python=tuple(sys.version_info[:3]),
         fractal_server=fractal_server.__VERSION__,
     )
     return versions

fractal_server/app/schemas/user.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from fastapi_users import schemas
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -41,8 +39,8 @@ class UserRead(schemas.BaseUser[int]):
         username:
     """
 
-    username: Optional[str] = None
-    group_ids_names: Optional[list[tuple[int, str]]] = None
+    username: str | None = None
+    group_ids_names: list[tuple[int, str]] | None = None
     oauth_accounts: list[OAuthAccountRead]
 
 

fractal_server/app/schemas/user_group.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -33,7 +32,7 @@ class UserGroupRead(BaseModel):
     id: int
     name: str
     timestamp_created: AwareDatetime
-    user_ids: Optional[list[int]] = None
+    user_ids: list[int] | None = None
     viewer_paths: list[str]
 
     @field_serializer("timestamp_created")

fractal_server/app/schemas/user_settings.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import field_validator
@@ -23,21 +21,21 @@ class UserSettingsRead(BaseModel):
     """
 
     id: int
-    ssh_host: Optional[str] = None
-    ssh_username: Optional[str] = None
-    ssh_private_key_path: Optional[str] = None
-    ssh_tasks_dir: Optional[str] = None
-    ssh_jobs_dir: Optional[str] = None
-    slurm_user: Optional[str] = None
+    ssh_host: str | None = None
+    ssh_username: str | None = None
+    ssh_private_key_path: str | None = None
+    ssh_tasks_dir: str | None = None
+    ssh_jobs_dir: str | None = None
+    slurm_user: str | None = None
     slurm_accounts: list[str]
-    project_dir: Optional[str] = None
+    project_dir: str | None = None
 
 
 class UserSettingsReadStrict(BaseModel):
-    slurm_user: Optional[str] = None
+    slurm_user: str | None = None
     slurm_accounts: list[str]
-    ssh_username: Optional[str] = None
-    project_dir: Optional[str] = None
+    ssh_username: str | None = None
+    project_dir: str | None = None
 
 
 class UserSettingsUpdate(BaseModel):
@@ -47,14 +45,14 @@ class UserSettingsUpdate(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    ssh_host: Optional[NonEmptyStr] = None
-    ssh_username: Optional[NonEmptyStr] = None
-    ssh_private_key_path: Optional[AbsolutePathStr] = None
-    ssh_tasks_dir: Optional[AbsolutePathStr] = None
-    ssh_jobs_dir: Optional[AbsolutePathStr] = None
-    slurm_user: Optional[NonEmptyStr] = None
-    slurm_accounts: Optional[ListUniqueNonEmptyString] = None
-    project_dir: Optional[AbsolutePathStr] = None
+    ssh_host: NonEmptyStr | None = None
+    ssh_username: NonEmptyStr | None = None
+    ssh_private_key_path: AbsolutePathStr | None = None
+    ssh_tasks_dir: AbsolutePathStr | None = None
+    ssh_jobs_dir: AbsolutePathStr | None = None
+    slurm_user: NonEmptyStr | None = None
+    slurm_accounts: ListUniqueNonEmptyString | None = None
+    project_dir: AbsolutePathStr | None = None
 
     @field_validator("project_dir", mode="after")
     @classmethod
@@ -66,4 +64,4 @@ class UserSettingsUpdate(BaseModel):
 
 class UserSettingsUpdateStrict(BaseModel):
     model_config = ConfigDict(extra="forbid")
-    slurm_accounts: Optional[ListUniqueNonEmptyString] = None
+    slurm_accounts: ListUniqueNonEmptyString | None = None

fractal_server/app/schemas/v2/dataset.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -19,7 +18,7 @@ class DatasetCreateV2(BaseModel):
 
     name: NonEmptyStr
 
-    zarr_dir: Optional[ZarrDirStr] = None
+    zarr_dir: ZarrDirStr | None = None
 
     attribute_filters: AttributeFilters = Field(default_factory=dict)
 
@@ -44,7 +43,7 @@ class DatasetUpdateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     name: NonEmptyStr = None
-    zarr_dir: Optional[ZarrDirStr] = None
+    zarr_dir: ZarrDirStr | None = None
 
 
 class DatasetImportV2(BaseModel):