fractal-server 2.15.0a5__tar.gz → 2.15.2__tar.gz

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.0a5
+Version: 2.15.2
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.15.2/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.15.2"

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/db/__init__.py

@@ -45,13 +45,11 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        engine_kwargs_async = {"pool_pre_ping": True}
-
         cls._engine_async = create_async_engine(
             settings.DATABASE_ASYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
-            **engine_kwargs_async,
+            pool_pre_ping=True,
         )
         cls._async_session_maker = sessionmaker(
             cls._engine_async,
@@ -65,13 +63,11 @@ class DB:
         settings = Inject(get_settings)
         settings.check_db()
 
-        engine_kwargs_sync = {}
-
         cls._engine_sync = create_engine(
             settings.DATABASE_SYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
-            **engine_kwargs_sync,
+            pool_pre_ping=True,
         )
 
         cls._sync_session_maker = sessionmaker(

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/security.py

@@ -15,8 +15,8 @@ from typing import Optional
 from pydantic import ConfigDict
 from pydantic import EmailStr
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -124,5 +124,5 @@ class UserGroup(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
     viewer_paths: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/user_settings.py

@@ -1,5 +1,5 @@
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -25,7 +25,7 @@ class UserSettings(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     slurm_accounts: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
     ssh_host: str | None = None
     ssh_username: str | None = None

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/v2/dataset.py

@@ -3,8 +3,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -24,7 +24,7 @@ class DatasetV2(SQLModel, table=True):
     )
 
     history: list[dict[str, Any]] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
 
     timestamp_created: datetime = Field(
@@ -34,7 +34,7 @@ class DatasetV2(SQLModel, table=True):
 
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
 
     @property

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/v2/job.py

@@ -3,8 +3,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -31,13 +31,13 @@ class JobV2(SQLModel, table=True):
     slurm_account: str | None = None
 
     dataset_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
     workflow_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
     project_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
 
     worker_init: str | None = None
@@ -57,8 +57,8 @@ class JobV2(SQLModel, table=True):
     log: str | None = None
 
     attribute_filters: AttributeFilters = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )
     type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/v2/task.py

@@ -1,7 +1,8 @@
 from typing import Any
 
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -33,8 +34,8 @@ class TaskV2(SQLModel, table=True):
     docs_info: str | None = None
     docs_link: str | None = None
 
-    input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
-    output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
+    input_types: dict[str, bool] = Field(sa_column=Column(JSONB), default={})
+    output_types: dict[str, bool] = Field(sa_column=Column(JSONB), default={})
 
     taskgroupv2_id: int = Field(foreign_key="taskgroupv2.id")
 
@@ -42,5 +43,5 @@ class TaskV2(SQLModel, table=True):
     modality: str | None = None
     authors: str | None = None
     tags: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/v2/task_group.py

@@ -2,8 +2,8 @@ from datetime import datetime
 from datetime import timezone
 
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -35,7 +35,7 @@ class TaskGroupV2(SQLModel, table=True):
     pip_extras: str | None = None
     pinned_package_versions: dict[str, str] = Field(
         sa_column=Column(
-            JSON,
+            JSONB,
             server_default="{}",
             default={},
             nullable=True,

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/models/v2/workflowtask.py

@@ -2,7 +2,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -24,14 +25,14 @@ class WorkflowTaskV2(SQLModel, table=True):
         sa_column=Column(JSON), default=None
     )
     args_parallel: dict[str, Any] | None = Field(
-        sa_column=Column(JSON), default=None
+        sa_column=Column(JSONB), default=None
     )
     args_non_parallel: dict[str, Any] | None = Field(
-        sa_column=Column(JSON), default=None
+        sa_column=Column(JSONB), default=None
     )
 
     type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )
 
     # Task

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/routes/admin/v2/job.py

@@ -156,8 +156,15 @@ async def update_job(
             detail=f"Cannot set job status to {job_update.status}",
         )
 
+    timestamp = get_timestamp()
     setattr(job, "status", job_update.status)
-    setattr(job, "end_timestamp", get_timestamp())
+    setattr(job, "end_timestamp", timestamp)
+    setattr(
+        job,
+        "log",
+        f"{job.log or ''}\nThis job was manually marked as "
+        f"'{JobStatusTypeV2.FAILED}' by an admin ({timestamp.isoformat()}).",
+    )
     await db.commit()
     await db.refresh(job)
     await db.close()

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/routes/api/v2/_aux_functions.py

@@ -325,6 +325,24 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
     return stm
 
 
+async def _workflow_has_submitted_job(
+    workflow_id: int,
+    db: AsyncSession,
+) -> bool:
+
+    res = await db.execute(
+        select(JobV2.id)
+        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+        .where(JobV2.workflow_id == workflow_id)
+        .limit(1)
+    )
+    submitted_jobs = res.scalar_one_or_none()
+    if submitted_jobs is not None:
+        return True
+
+    return False
+
+
 async def _workflow_insert_task(
     *,
     workflow_id: int,

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/routes/api/v2/workflow.py

@@ -22,6 +22,7 @@ from ._aux_functions import _check_workflow_exists
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import _workflow_has_submitted_job
 from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -146,6 +147,18 @@ async def update_workflow(
 
     for key, value in patch.model_dump(exclude_unset=True).items():
         if key == "reordered_workflowtask_ids":
+
+            if await _workflow_has_submitted_job(
+                workflow_id=workflow_id, db=db
+            ):
+                raise HTTPException(
+                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                    detail=(
+                        "Cannot re-order WorkflowTasks while a Job is running "
+                        "for this Workflow."
+                    ),
+                )
+
             current_workflowtask_ids = [
                 wftask.id for wftask in workflow.task_list
             ]

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/routes/api/v2/workflowtask.py

@@ -10,6 +10,7 @@ from ....db import AsyncSession
 from ....db import get_async_db
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _get_workflow_task_check_owner
+from ._aux_functions import _workflow_has_submitted_job
 from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
@@ -224,6 +225,15 @@ async def delete_workflowtask(
         db=db,
     )
 
+    if await _workflow_has_submitted_job(workflow_id=workflow_id, db=db):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot delete a WorkflowTask while a Job is running for this "
+                "Workflow."
+            ),
+        )
+
     # Delete WorkflowTask
     await db.delete(db_workflow_task)
     await db.commit()

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -48,6 +48,8 @@ class _SlurmConfigSet(BaseModel):
         constraint:
         gres:
         time:
+        exclude:
+        nodelist:
         account:
         extra_lines:
     """
@@ -59,6 +61,8 @@ class _SlurmConfigSet(BaseModel):
     mem: int | str | None = None
     constraint: str | None = None
     gres: str | None = None
+    exclude: str | None = None
+    nodelist: str | None = None
     time: str | None = None
     account: str | None = None
     extra_lines: list[str] | None = None
@@ -227,6 +231,8 @@ class SlurmConfig(BaseModel):
         account: Corresponds to SLURM option.
         gpus: Corresponds to SLURM option.
         time: Corresponds to SLURM option (WARNING: not fully supported).
+        nodelist: Corresponds to SLURM option.
+        exclude: Corresponds to SLURM option.
         prefix: Prefix of configuration lines in SLURM submission scripts.
         shebang_line: Shebang line for SLURM submission scripts.
         extra_lines: Additional lines to include in SLURM submission scripts.
@@ -268,6 +274,8 @@ class SlurmConfig(BaseModel):
     gpus: str | None = None
     time: str | None = None
     account: str | None = None
+    nodelist: str | None = None
+    exclude: str | None = None
 
     # Free-field attribute for extra lines to be added to the SLURM job
     # preamble
@@ -361,6 +369,8 @@ class SlurmConfig(BaseModel):
             "gpus",
             "time",
             "account",
+            "exclude",
+            "nodelist",
         ]:
             value = getattr(self, key)
             if value is not None:

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -137,6 +137,34 @@ class BaseSlurmRunner(BaseRunner):
     def run_squeue(self, *, job_ids: list[str], **kwargs) -> str:
         raise NotImplementedError("Implement in child class.")
 
+    def _is_squeue_error_recoverable(self, exception: BaseException) -> True:
+        """
+        Determine whether a `squeue` error is considered recoverable.
+
+        A _recoverable_ error is one which will disappear after some time,
+        without any specific action from the `fractal-server` side.
+
+        Note: if this function returns `True` for an error that does not
+        actually recover, this leads to an infinite loop  where
+        `fractal-server` keeps polling `squeue` information forever.
+
+        More info at
+        https://github.com/fractal-analytics-platform/fractal-server/issues/2682
+
+        Args:
+            exception: The exception raised by `self.run_squeue`.
+        Returns:
+            Whether the error is considered recoverable.
+        """
+        str_exception = str(exception)
+        if (
+            "slurm_load_jobs" in str_exception
+            and "Socket timed out on send/recv operation" in str_exception
+        ):
+            return True
+        else:
+            return False
+
     def _get_finished_jobs(self, job_ids: list[str]) -> set[str]:
         #  If there is no Slurm job to check, return right away
         if not job_ids:
@@ -161,12 +189,26 @@ class BaseSlurmRunner(BaseRunner):
                         {stdout.split()[0]: stdout.split()[1]}
                     )
                 except Exception as e:
-                    logger.warning(
-                        "[_get_finished_jobs] `squeue` failed for "
-                        f"{job_id=}, mark job as completed. "
+                    msg = (
+                        f"[_get_finished_jobs] `squeue` failed for {job_id=}. "
                         f"Original error: {str(e)}."
                     )
-                    slurm_statuses.update({str(job_id): "COMPLETED"})
+                    logger.warning(msg)
+                    if self._is_squeue_error_recoverable(e):
+                        logger.warning(
+                            "[_get_finished_jobs] Recoverable `squeue` "
+                            f"error - mark {job_id=} as FRACTAL_UNDEFINED and"
+                            " retry later."
+                        )
+                        slurm_statuses.update(
+                            {str(job_id): "FRACTAL_UNDEFINED"}
+                        )
+                    else:
+                        logger.warning(
+                            "[_get_finished_jobs] Non-recoverable `squeue`"
+                            f"error - mark {job_id=} as completed."
+                        )
+                        slurm_statuses.update({str(job_id): "COMPLETED"})
 
         # If a job is not in `squeue` output, mark it as completed.
         finished_jobs = {
@@ -182,33 +224,53 @@ class BaseSlurmRunner(BaseRunner):
     def _mkdir_remote_folder(self, folder: str) -> None:
         raise NotImplementedError("Implement in child class.")
 
-    def _submit_single_sbatch(
+    def _enrich_slurm_config(
         self,
-        *,
-        base_command: str,
-        slurm_job: SlurmJob,
         slurm_config: SlurmConfig,
-    ) -> str:
-        logger.debug("[_submit_single_sbatch] START")
+    ) -> SlurmConfig:
+        """
+        Return an enriched `SlurmConfig` object
 
-        # Include SLURM account in `slurm_config`. Note: we make this change
-        # here, rather than exposing a new argument of `get_slurm_config`,
-        # because it's a backend-specific argument while `get_slurm_config` has
-        # a generic interface.
+        Include `self.account` and `self.common_script_lines` into a
+        `SlurmConfig` object. Extracting this logic into an independent
+        class method is useful to fix issue #2659 (which was due to
+        performing this same operation multiple times rather than once).
+
+        Args:
+            slurm_config: The original `SlurmConfig` object.
+
+        Returns:
+            A new, up-to-date, `SlurmConfig` object.
+        """
+
+        new_slurm_config = slurm_config.model_copy()
+
+        # Include SLURM account in `slurm_config`.
         if self.slurm_account is not None:
-            slurm_config.account = self.slurm_account
+            new_slurm_config.account = self.slurm_account
 
         # Include common_script_lines in extra_lines
         if len(self.common_script_lines) > 0:
             logger.debug(
                 f"Add {self.common_script_lines} to "
-                f"{slurm_config.extra_lines=}."
+                f"{new_slurm_config.extra_lines=}."
             )
-            current_extra_lines = slurm_config.extra_lines or []
-            slurm_config.extra_lines = (
+            current_extra_lines = new_slurm_config.extra_lines or []
+            new_slurm_config.extra_lines = (
                 current_extra_lines + self.common_script_lines
             )
 
+        return new_slurm_config
+
+    def _submit_single_sbatch(
+        self,
+        *,
+        base_command: str,
+        slurm_job: SlurmJob,
+        slurm_config: SlurmConfig,
+    ) -> str:
+        logger.debug("[_submit_single_sbatch] START")
+
         for task in slurm_job.tasks:
             # Write input file
             if self.slurm_runner_type == "ssh":
@@ -508,6 +570,9 @@ class BaseSlurmRunner(BaseRunner):
         user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
+
+        config = self._enrich_slurm_config(config)
+
         try:
             workdir_local = task_files.wftask_subfolder_local
             workdir_remote = task_files.wftask_subfolder_remote
@@ -649,6 +714,8 @@ class BaseSlurmRunner(BaseRunner):
         input images, while for compound tasks these can differ.
         """
 
+        config = self._enrich_slurm_config(config)
+
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
         try:
             if self.is_shutdown():

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -125,7 +125,14 @@ def get_slurm_config_internal(
             )
             logger.error(error_msg)
             raise SlurmConfigError(error_msg)
-        for key in ["time", "gres", "gpus", "constraint"]:
+        for key in [
+            "time",
+            "gres",
+            "gpus",
+            "constraint",
+            "nodelist",
+            "exclude",
+        ]:
             value = wftask_meta.get(key, None)
             if value is not None:
                 slurm_dict[key] = value

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/runner/executors/slurm_common/remote.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import logging
 import os
 import sys
 
@@ -32,7 +31,6 @@ def worker(
     # Create output folder, if missing
     out_dir = os.path.dirname(out_fname)
     if not os.path.exists(out_dir):
-        logging.debug(f"_slurm.remote.worker: create {out_dir=}")
         os.mkdir(out_dir)
 
     # Execute the job and capture exceptions
@@ -40,10 +38,8 @@ def worker(
         with open(in_fname) as f:
             input_data = json.load(f)
 
-        server_python_version = input_data["python_version"]
-        server_fractal_server_version = input_data["fractal_server_version"]
-
         # Fractal-server version must be identical
+        server_fractal_server_version = input_data["fractal_server_version"]
         worker_fractal_server_version = __VERSION__
         if worker_fractal_server_version != server_fractal_server_version:
             raise FractalVersionMismatch(
@@ -51,11 +47,16 @@ def worker(
                 f"{worker_fractal_server_version=}"
             )
 
-        # Python version mismatch only raises a warning
-        worker_python_version = tuple(sys.version_info[:3])
+        # Get `worker_python_version` as a `list` since this is the type of
+        # `server_python_version` after a JSON dump/load round trip.
+        worker_python_version = list(sys.version_info[:3])
+
+        # Print a warning for Python version mismatch
+        server_python_version = input_data["python_version"]
         if worker_python_version != server_python_version:
             if worker_python_version[:2] != server_python_version[:2]:
-                logging.warning(
+                print(
+                    "WARNING: "
                     f"{server_python_version=} but {worker_python_version=}."
                 )
 
@@ -116,7 +117,6 @@ if __name__ == "__main__":
         required=True,
     )
     parsed_args = parser.parse_args()
-    logging.debug(f"{parsed_args=}")
 
     kwargs = dict(
         in_fname=parsed_args.input_file,

{fractal_server-2.15.0a5 → fractal_server-2.15.2}/fractal_server/app/runner/v2/_slurm_ssh.py

@@ -20,7 +20,6 @@ from pathlib import Path
 from ....ssh._fabric import FractalSSH
 from ...models.v2 import DatasetV2
 from ...models.v2 import WorkflowV2
-from ..exceptions import JobExecutionError
 from ..executors.slurm_common.get_slurm_config import get_slurm_config
 from ..executors.slurm_ssh.runner import SlurmSSHRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
@@ -64,18 +63,6 @@ def process_workflow(
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
-    # Create main remote folder
-    try:
-        fractal_ssh.mkdir(folder=str(workflow_dir_remote))
-        logger.info(f"Created {str(workflow_dir_remote)} via SSH.")
-    except Exception as e:
-        error_msg = (
-            f"Could not create {str(workflow_dir_remote)} via SSH.\n"
-            f"Original error: {str(e)}."
-        )
-        logger.error(error_msg)
-        raise JobExecutionError(info=error_msg)
-
     with SlurmSSHRunner(
         fractal_ssh=fractal_ssh,
         root_dir_local=workflow_dir_local,