fractal-server 2.12.1__py3-none-any.whl → 2.13.1__py3-none-any.whl

fractal_server/app/routes/api/v2/project.py

@@ -62,7 +62,7 @@ async def create_project(
         project_name=project.name, user_id=user.id, db=db
     )
 
-    db_project = ProjectV2(**project.dict())
+    db_project = ProjectV2(**project.model_dump())
     db_project.user_list.append(user)
 
     db.add(db_project)
@@ -106,7 +106,7 @@ async def update_project(
             project_name=project_update.name, user_id=user.id, db=db
         )
 
-    for key, value in project_update.dict(exclude_unset=True).items():
+    for key, value in project_update.model_dump(exclude_unset=True).items():
         setattr(project, key, value)
 
     await db.commit()

fractal_server/app/routes/api/v2/status.py

@@ -154,7 +154,7 @@ async def get_workflowtask_status(
         if wf_task_status is None:
             # If a wftask ID was not found, ignore it and continue
             continue
-        clean_workflow_tasks_status_dict[wf_task.id] = wf_task_status
+        clean_workflow_tasks_status_dict[str(wf_task.id)] = wf_task_status
         if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
             # Starting from the beginning of `workflow.task_list`, stop the
             # first time that you hit a failed job

fractal_server/app/routes/api/v2/submit.py

@@ -165,11 +165,15 @@ async def apply_workflow(
         # The 'filters' field is not supported any more but still exists as a
         # database column, therefore we manually exclude it from dumps.
         dataset_dump=json.loads(
-            dataset.json(exclude={"images", "history", "filters"})
+            dataset.model_dump_json(exclude={"images", "history", "filters"})
         ),
-        workflow_dump=json.loads(workflow.json(exclude={"task_list"})),
-        project_dump=json.loads(project.json(exclude={"user_list"})),
-        **job_create.dict(),
+        workflow_dump=json.loads(
+            workflow.model_dump_json(exclude={"task_list"})
+        ),
+        project_dump=json.loads(
+            project.model_dump_json(exclude={"user_list"})
+        ),
+        **job_create.model_dump(),
     )
 
     db.add(job)
@@ -202,8 +206,6 @@ async def apply_workflow(
     # Define user-side job directory
     if FRACTAL_RUNNER_BACKEND == "local":
         WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
-    elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-        WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
     elif FRACTAL_RUNNER_BACKEND == "slurm":
         WORKFLOW_DIR_REMOTE = cache_dir / WORKFLOW_DIR_LOCAL.name
     elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
@@ -237,6 +239,7 @@ async def apply_workflow(
         workflow_id=workflow.id,
         dataset_id=dataset.id,
         job_id=job.id,
+        user_id=user.id,
         user_settings=user_settings,
         worker_init=job.worker_init,
         slurm_user=user_settings.slurm_user,

fractal_server/app/routes/api/v2/task.py

@@ -107,7 +107,7 @@ async def patch_task(
     db_task = await _get_task_full_access(
         task_id=task_id, user_id=user.id, db=db
     )
-    update = task_update.dict(exclude_unset=True)
+    update = task_update.model_dump(exclude_unset=True)
 
     # Forbid changes that set a previously unset command
     if db_task.type == "non_parallel" and "command_parallel" in update:
@@ -182,7 +182,8 @@ async def create_task(
         )
 
     # Add task
-    db_task = TaskV2(**task.dict(), type=task_type)
+
+    db_task = TaskV2(**task.model_dump(exclude_unset=True), type=task_type)
     pkg_name = db_task.name
     await _verify_non_duplication_user_constraint(
         db=db, pkg_name=pkg_name, user_id=user.id, version=db_task.version
@@ -206,6 +207,7 @@ async def create_task(
     await db.commit()
     await db.refresh(db_task)
     await db.close()
+
     return db_task
 
 

fractal_server/app/routes/api/v2/task_collection.py

@@ -13,7 +13,7 @@ from fastapi import Response
 from fastapi import status
 from fastapi import UploadFile
 from pydantic import BaseModel
-from pydantic import root_validator
+from pydantic import model_validator
 from pydantic import ValidationError
 from sqlmodel import select
 
@@ -68,7 +68,8 @@ class CollectionRequestData(BaseModel):
     file: Optional[UploadFile] = None
     origin: TaskGroupV2OriginEnum
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def validate_data(cls, values):
         file = values.get("file")
         package = values.get("task_collect").package

fractal_server/app/routes/api/v2/task_group.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import Optional
 
 from fastapi import APIRouter
@@ -6,6 +5,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
 
@@ -23,7 +23,6 @@ from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
-from fractal_server.app.routes.aux import _raise_if_naive_datetime
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
@@ -43,13 +42,11 @@ async def get_task_group_activity_list(
     pkg_name: Optional[str] = None,
     status: Optional[TaskGroupActivityStatusV2] = None,
     action: Optional[TaskGroupActivityActionV2] = None,
-    timestamp_started_min: Optional[datetime] = None,
+    timestamp_started_min: Optional[AwareDatetime] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskGroupActivityV2Read]:
 
-    _raise_if_naive_datetime(timestamp_started_min)
-
     stm = select(TaskGroupActivityV2).where(
         TaskGroupActivityV2.user_id == user.id
     )
@@ -223,7 +220,7 @@ async def patch_task_group(
         db=db,
     )
     if (
-        "user_group_id" in task_group_update.dict(exclude_unset=True)
+        "user_group_id" in task_group_update.model_dump(exclude_unset=True)
         and task_group_update.user_group_id != task_group.user_group_id
     ):
         await _verify_non_duplication_group_constraint(
@@ -232,7 +229,7 @@ async def patch_task_group(
             version=task_group.version,
             user_group_id=task_group_update.user_group_id,
         )
-    for key, value in task_group_update.dict(exclude_unset=True).items():
+    for key, value in task_group_update.model_dump(exclude_unset=True).items():
         if (key == "user_group_id") and (value is not None):
             await _verify_user_belongs_to_group(
                 user_id=user.id, user_group_id=value, db=db

fractal_server/app/routes/api/v2/workflow.py

@@ -82,7 +82,7 @@ async def create_workflow(
         name=workflow.name, project_id=project_id, db=db
     )
 
-    db_workflow = WorkflowV2(project_id=project_id, **workflow.dict())
+    db_workflow = WorkflowV2(project_id=project_id, **workflow.model_dump())
     db.add(db_workflow)
     await db.commit()
     await db.refresh(db_workflow)
@@ -149,7 +149,7 @@ async def update_workflow(
             name=patch.name, project_id=project_id, db=db
         )
 
-    for key, value in patch.dict(exclude_unset=True).items():
+    for key, value in patch.model_dump(exclude_unset=True).items():
         if key == "reordered_workflowtask_ids":
             current_workflowtask_ids = [
                 wftask.id for wftask in workflow.task_list
@@ -262,7 +262,7 @@ async def export_worfklow(
     wf_task_list = []
     for wftask in workflow.task_list:
         task_group = await db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
-        wf_task_list.append(wftask.dict())
+        wf_task_list.append(wftask.model_dump())
         wf_task_list[-1]["task"] = dict(
             pkg_name=task_group.pkg_name,
             version=task_group.version,

fractal_server/app/routes/api/v2/workflow_import.py

@@ -321,7 +321,7 @@ async def import_workflow(
                 detail=f"Could not find a task matching with {wf_task.task}.",
             )
         new_wf_task = WorkflowTaskCreateV2(
-            **wf_task.dict(exclude_none=True, exclude={"task"})
+            **wf_task.model_dump(exclude_none=True, exclude={"task"})
         )
         list_wf_tasks.append(new_wf_task)
         list_task_ids.append(task_id)
@@ -336,7 +336,7 @@ async def import_workflow(
     # Create new Workflow
     db_workflow = WorkflowV2(
         project_id=project_id,
-        **workflow_import.dict(exclude_none=True, exclude={"task_list"}),
+        **workflow_import.model_dump(exclude_none=True, exclude={"task_list"}),
     )
     db.add(db_workflow)
     await db.commit()
@@ -345,7 +345,7 @@ async def import_workflow(
     # Insert task into the workflow
     for ind, new_wf_task in enumerate(list_wf_tasks):
         await _workflow_insert_task(
-            **new_wf_task.dict(),
+            **new_wf_task.model_dump(),
             workflow_id=db_workflow.id,
             task_id=list_task_ids[ind],
             db=db,

fractal_server/app/routes/api/v2/workflowtask.py

@@ -281,7 +281,9 @@ async def update_workflowtask(
             ),
         )
 
-    for key, value in workflow_task_update.dict(exclude_unset=True).items():
+    for key, value in workflow_task_update.model_dump(
+        exclude_unset=True
+    ).items():
         if key == "args_parallel":
             # Get default arguments via a Task property method
             actual_args = deepcopy(value)

fractal_server/app/routes/auth/_aux_auth.py

@@ -58,11 +58,14 @@ async def _get_single_user_with_groups(
         group_ids_names.insert(0, default_group)
     else:
         pass
+    oauth_accounts = [
+        oauth_account.model_dump() for oauth_account in user.oauth_accounts
+    ]
 
     return UserRead(
         **user.model_dump(),
         group_ids_names=group_ids_names,
-        oauth_accounts=user.oauth_accounts,
+        oauth_accounts=oauth_accounts,
     )
 
 

fractal_server/app/routes/auth/current_user.py

@@ -57,14 +57,14 @@ async def patch_current_user(
     Note: a user cannot patch their own password (as enforced within the
     `UserUpdateStrict` schema).
     """
-    update = UserUpdate(**user_update.dict(exclude_unset=True))
+    update = UserUpdate(**user_update.model_dump(exclude_unset=True))
 
     # NOTE: here it would be relevant to catch an `InvalidPasswordException`
     # (from `fastapi_users.exceptions`), if we were to allow users change
     # their own password
 
     user = await user_manager.update(update, current_user, safe=True)
-    validated_user = schemas.model_validate(UserOAuth, user)
+    validated_user = schemas.model_validate(UserOAuth, user.model_dump())
 
     patched_user = await db.get(
         UserOAuth, validated_user.id, populate_existing=True
@@ -82,7 +82,6 @@ async def get_current_user_settings(
     current_user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsReadStrict:
-
     verify_user_has_settings(current_user)
     user_settings = await db.get(UserSettings, current_user.user_settings_id)
     return user_settings
@@ -96,13 +95,12 @@ async def patch_current_user_settings(
     current_user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsReadStrict:
-
     verify_user_has_settings(current_user)
     current_user_settings = await db.get(
         UserSettings, current_user.user_settings_id
     )
 
-    for k, v in settings_update.dict(exclude_unset=True).items():
+    for k, v in settings_update.model_dump(exclude_unset=True).items():
         setattr(current_user_settings, k, v)
 
     db.add(current_user_settings)

fractal_server/app/routes/auth/group.py

@@ -194,7 +194,7 @@ async def patch_user_settings_bulk(
         .where(LinkUserGroup.group_id == group_id)
     )
     settings_list = res.scalars().all()
-    update = settings_update.dict(exclude_unset=True)
+    update = settings_update.model_dump(exclude_unset=True)
     for settings in settings_list:
         for k, v in update.items():
             setattr(settings, k, v)

fractal_server/app/routes/auth/users.py

@@ -75,7 +75,7 @@ async def patch_user(
             safe=False,
             request=None,
         )
-        validated_user = schemas.model_validate(UserOAuth, user)
+        validated_user = schemas.model_validate(UserOAuth, user.model_dump())
         patched_user = await db.get(
             UserOAuth, validated_user.id, populate_existing=True
         )
@@ -139,7 +139,6 @@ async def set_user_groups(
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserRead:
-
     # Preliminary check that all objects exist in the db
     user = await _user_or_404(user_id=user_id, db=db)
     target_group_ids = user_update.group_ids
@@ -209,7 +208,6 @@ async def get_user_settings(
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> UserSettingsRead:
-
     user = await _user_or_404(user_id=user_id, db=db)
     verify_user_has_settings(user)
     user_settings = await db.get(UserSettings, user.user_settings_id)
@@ -229,7 +227,7 @@ async def patch_user_settings(
     verify_user_has_settings(user)
     user_settings = await db.get(UserSettings, user.user_settings_id)
 
-    for k, v in settings_update.dict(exclude_unset=True).items():
+    for k, v in settings_update.model_dump(exclude_unset=True).items():
         setattr(user_settings, k, v)
 
     db.add(user_settings)

fractal_server/app/routes/aux/__init__.py

@@ -1,20 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from fastapi import HTTPException
-from fastapi import status
-
-
-def _raise_if_naive_datetime(*timestamps: tuple[Optional[datetime]]) -> None:
-    """
-    Raise 422 if any not-null argument is a naive `datetime` object:
-    https://docs.python.org/3/library/datetime.html#determining-if-an-object-is-aware-or-naive
-    """
-    for timestamp in filter(None, timestamps):
-        if (timestamp.tzinfo is None) or (
-            timestamp.tzinfo.utcoffset(timestamp) is None
-        ):
-            raise HTTPException(
-                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                detail=f"{timestamp=} is naive. You must provide a timezone.",
-            )

fractal_server/app/routes/aux/_runner.py

@@ -6,7 +6,7 @@ from ....syringe import Inject
 
 
 def _backend_supports_shutdown(backend: str) -> bool:
-    if backend in ["slurm", "slurm_ssh", "local_experimental"]:
+    if backend in ["slurm", "slurm_ssh"]:
         return True
     else:
         return False

fractal_server/app/routes/aux/validate_user_settings.py

@@ -1,6 +1,5 @@
 from fastapi import HTTPException
 from fastapi import status
-from pydantic import BaseModel
 from pydantic import ValidationError
 
 from fractal_server.app.db import AsyncSession
@@ -55,7 +54,7 @@ async def validate_user_settings(
         UserSettingsValidationModel = SlurmSudoUserSettings
     else:
         # For other backends, we don't validate anything
-        UserSettingsValidationModel = BaseModel
+        return user_settings
 
     try:
         UserSettingsValidationModel(**user_settings.model_dump())

fractal_server/app/runner/executors/_job_states.py

@@ -0,0 +1,13 @@
+# https://slurm.schedmd.com/squeue.html#lbAG
+STATES_FINISHED = {
+    "BOOT_FAIL",
+    "CANCELLED",
+    "COMPLETED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "SPECIAL_EXIT",
+    "TIMEOUT",
+}

fractal_server/app/runner/executors/slurm/_slurm_config.py

@@ -18,9 +18,9 @@ from typing import Optional
 from typing import Union
 
 from pydantic import BaseModel
-from pydantic import Extra
+from pydantic import ConfigDict
 from pydantic import Field
-from pydantic.error_wrappers import ValidationError
+from pydantic import ValidationError
 
 from .....config import get_settings
 from .....logger import set_logger
@@ -37,7 +37,7 @@ class SlurmConfigError(ValueError):
     pass
 
 
-class _SlurmConfigSet(BaseModel, extra=Extra.forbid):
+class _SlurmConfigSet(BaseModel):
     """
     Options that can be set in `FRACTAL_SLURM_CONFIG_FILE` for the default/gpu
     SLURM config. Only used as part of `SlurmConfigFile`.
@@ -54,19 +54,21 @@ class _SlurmConfigSet(BaseModel, extra=Extra.forbid):
         extra_lines:
     """
 
-    partition: Optional[str]
-    cpus_per_task: Optional[int]
-    mem: Optional[Union[int, str]]
-    constraint: Optional[str]
-    gres: Optional[str]
-    time: Optional[str]
-    account: Optional[str]
-    extra_lines: Optional[list[str]]
-    pre_submission_commands: Optional[list[str]]
-    gpus: Optional[str]
+    model_config = ConfigDict(extra="forbid")
 
+    partition: Optional[str] = None
+    cpus_per_task: Optional[int] = None
+    mem: Optional[Union[int, str]] = None
+    constraint: Optional[str] = None
+    gres: Optional[str] = None
+    time: Optional[str] = None
+    account: Optional[str] = None
+    extra_lines: Optional[list[str]] = None
+    pre_submission_commands: Optional[list[str]] = None
+    gpus: Optional[str] = None
 
-class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
+
+class _BatchingConfigSet(BaseModel):
     """
     Options that can be set in `FRACTAL_SLURM_CONFIG_FILE` to configure the
     batching strategy (that is, how to combine several tasks in a single SLURM
@@ -83,6 +85,8 @@ class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
         max_num_jobs:
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     target_cpus_per_job: int
     max_cpus_per_job: int
     target_mem_per_job: Union[int, str]
@@ -91,7 +95,7 @@ class _BatchingConfigSet(BaseModel, extra=Extra.forbid):
     max_num_jobs: int
 
 
-class SlurmConfigFile(BaseModel, extra=Extra.forbid):
+class SlurmConfigFile(BaseModel):
     """
     Specifications for the content of `FRACTAL_SLURM_CONFIG_FILE`
 
@@ -136,10 +140,12 @@ class SlurmConfigFile(BaseModel, extra=Extra.forbid):
             directory.
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     default_slurm_config: _SlurmConfigSet
-    gpu_slurm_config: Optional[_SlurmConfigSet]
+    gpu_slurm_config: Optional[_SlurmConfigSet] = None
     batching_config: _BatchingConfigSet
-    user_local_exports: Optional[dict[str, str]]
+    user_local_exports: Optional[dict[str, str]] = None
 
 
 def load_slurm_config_file(
@@ -196,7 +202,7 @@ def load_slurm_config_file(
     return obj
 
 
-class SlurmConfig(BaseModel, extra=Extra.forbid):
+class SlurmConfig(BaseModel):
     """
     Abstraction for SLURM parameters
 
@@ -247,6 +253,8 @@ class SlurmConfig(BaseModel, extra=Extra.forbid):
             command.
     """
 
+    model_config = ConfigDict(extra="forbid")
+
     # Required SLURM parameters (note that the integer attributes are those
     # that will need to scale up with the number of parallel tasks per job)
     partition: str

fractal_server/app/runner/executors/slurm/ssh/__init__.py

@@ -1,3 +0,0 @@
-from .executor import SlurmExecutor
-
-__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py

@@ -1,10 +1,8 @@
 import os
+import threading
 import time
 import traceback
 from itertools import count
-from typing import Callable
-
-from cfut import FileWaitThread
 
 from ......logger import set_logger
 from fractal_server.app.runner.exceptions import JobExecutionError
@@ -12,35 +10,46 @@ from fractal_server.app.runner.exceptions import JobExecutionError
 logger = set_logger(__name__)
 
 
-class FractalSlurmWaitThread(FileWaitThread):
+class FractalSlurmSSHWaitThread(threading.Thread):
     """
-    Overrides the original clusterfutures.FileWaitThread, so that:
-
-    1. Each jobid in the waiting list is associated to a tuple of filenames,
-       rather than a single one.
-    2. In the `check` method, we avoid output-file existence checks (which
-       would require `sudo -u user ls` calls), and we rather check for the
-       existence of the shutdown file. All the logic to check whether a job is
-       complete is deferred to the `cfut.slurm.jobs_finished` function.
-    3. There are additional attributes (...).
-
-    This class is based on clusterfutures 0.5. Original Copyright: 2022
-    Adrian Sampson, released under the MIT licence
+    Thread that monitors a pool of SLURM jobs
+
+    This class is a custom re-implementation of the waiting thread class from:
+
+    > clusterfutures <https://github.com/sampsyo/clusterfutures>
+    > Original Copyright
+    > Copyright 2021 Adrian Sampson <asampson@cs.washington.edu>
+    > License: MIT
+
+    Attributes:
+        shutdown_file:
+        shutdown_callback:
+        slurm_poll_interval:
+        jobs_finished_callback:
+        active_job_ids:
+        shutdown:
+        lock:
     """
 
     shutdown_file: str
-    shutdown_callback: Callable
-    jobs_finished_callback: Callable
+    shutdown_callback: callable
     slurm_poll_interval = 30
+    jobs_finished_callback: callable
     active_job_ids: list[str]
+    shutdown: bool
+    _lock: threading.Lock
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, callback: callable, interval=1):
         """
         Init method
 
         This method is executed on the main thread.
         """
-        super().__init__(*args, **kwargs)
+        threading.Thread.__init__(self, daemon=True)
+        self.callback = callback
+        self.interval = interval
+        self._lock = threading.Lock()
+        self.shutdown = False
         self.active_job_ids = []
 
     def wait(self, *, job_id: str):
@@ -53,7 +62,7 @@ class FractalSlurmWaitThread(FileWaitThread):
             error_msg = "Cannot call `wait` method after executor shutdown."
             logger.warning(error_msg)
             raise JobExecutionError(info=error_msg)
-        with self.lock:
+        with self._lock:
             self.active_job_ids.append(job_id)
 
     def check_shutdown(self):
@@ -109,7 +118,7 @@ class FractalSlurmWaitThread(FileWaitThread):
                     pass
                 return
             if ind % skip == 0:
-                with self.lock:
+                with self._lock:
                     try:
                         self.check_jobs()
                     except Exception:  # nosec

fractal_server/app/runner/executors/slurm/ssh/_slurm_job.py

@@ -1,8 +1,7 @@
+import uuid
 from pathlib import Path
 from typing import Optional
 
-from cfut.util import random_string
-
 from fractal_server.app.runner.executors.slurm._slurm_config import (
     SlurmConfig,
 )
@@ -106,9 +105,7 @@ class SlurmJob:
             )
         else:
             self.wftask_file_prefixes = wftask_file_prefixes
-        self.workerids = tuple(
-            random_string() for i in range(self.num_tasks_tot)
-        )
+        self.workerids = tuple(uuid.uuid4() for i in range(self.num_tasks_tot))
         self.slurm_config = slurm_config
 
     def get_clean_output_pickle_files(self) -> tuple[str, ...]: