fractal-server 2.15.0a4__tar.gz → 2.15.1__tar.gz

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.0a4
+Version: 2.15.1
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.15.1/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.15.1"

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/security.py

@@ -15,8 +15,8 @@ from typing import Optional
 from pydantic import ConfigDict
 from pydantic import EmailStr
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -124,5 +124,5 @@ class UserGroup(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
     viewer_paths: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/user_settings.py

@@ -1,5 +1,5 @@
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -25,7 +25,7 @@ class UserSettings(SQLModel, table=True):
 
     id: int | None = Field(default=None, primary_key=True)
     slurm_accounts: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
     ssh_host: str | None = None
     ssh_username: str | None = None

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/v2/dataset.py

@@ -3,8 +3,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -24,7 +24,7 @@ class DatasetV2(SQLModel, table=True):
     )
 
     history: list[dict[str, Any]] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
 
     timestamp_created: datetime = Field(
@@ -34,7 +34,7 @@ class DatasetV2(SQLModel, table=True):
 
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )
 
     @property

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/v2/job.py

@@ -3,8 +3,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -31,13 +31,13 @@ class JobV2(SQLModel, table=True):
     slurm_account: str | None = None
 
     dataset_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
     workflow_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
     project_dump: dict[str, Any] = Field(
-        sa_column=Column(JSON, nullable=False)
+        sa_column=Column(JSONB, nullable=False)
     )
 
     worker_init: str | None = None
@@ -57,8 +57,8 @@ class JobV2(SQLModel, table=True):
     log: str | None = None
 
     attribute_filters: AttributeFilters = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )
     type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/v2/task.py

@@ -1,7 +1,8 @@
 from typing import Any
 
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
@@ -33,8 +34,8 @@ class TaskV2(SQLModel, table=True):
     docs_info: str | None = None
     docs_link: str | None = None
 
-    input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
-    output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
+    input_types: dict[str, bool] = Field(sa_column=Column(JSONB), default={})
+    output_types: dict[str, bool] = Field(sa_column=Column(JSONB), default={})
 
     taskgroupv2_id: int = Field(foreign_key="taskgroupv2.id")
 
@@ -42,5 +43,5 @@ class TaskV2(SQLModel, table=True):
     modality: str | None = None
     authors: str | None = None
     tags: list[str] = Field(
-        sa_column=Column(JSON, server_default="[]", nullable=False)
+        sa_column=Column(JSONB, server_default="[]", nullable=False)
     )

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/v2/task_group.py

@@ -2,8 +2,8 @@ from datetime import datetime
 from datetime import timezone
 
 from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
-from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -35,7 +35,7 @@ class TaskGroupV2(SQLModel, table=True):
     pip_extras: str | None = None
     pinned_package_versions: dict[str, str] = Field(
         sa_column=Column(
-            JSON,
+            JSONB,
             server_default="{}",
             default={},
             nullable=True,

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/models/v2/workflowtask.py

@@ -2,7 +2,8 @@ from typing import Any
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
-from sqlalchemy.types import JSON
+from sqlalchemy.dialects.postgresql import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
@@ -24,14 +25,14 @@ class WorkflowTaskV2(SQLModel, table=True):
         sa_column=Column(JSON), default=None
     )
     args_parallel: dict[str, Any] | None = Field(
-        sa_column=Column(JSON), default=None
+        sa_column=Column(JSONB), default=None
     )
     args_non_parallel: dict[str, Any] | None = Field(
-        sa_column=Column(JSON), default=None
+        sa_column=Column(JSONB), default=None
     )
 
     type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
+        sa_column=Column(JSONB, nullable=False, server_default="{}")
     )
 
     # Task

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -333,6 +333,28 @@ async def _verify_non_duplication_group_constraint(
         )
 
 
+async def _verify_non_duplication_group_path(
+    path: str | None,
+    db: AsyncSession,
+) -> None:
+    """
+    Verify uniqueness of non-`None` `TaskGroupV2.path`
+    """
+    if path is None:
+        return
+    stm = select(TaskGroupV2.id).where(TaskGroupV2.path == path)
+    res = await db.execute(stm)
+    duplicate_ids = res.scalars().all()
+    if duplicate_ids:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Other TaskGroups already have {path=}: "
+                f"{sorted(duplicate_ids)}."
+            ),
+        )
+
+
 async def _add_warnings_to_workflow_tasks(
     wftask_list: list[WorkflowTaskV2], user_id: int, db: AsyncSession
 ) -> list[dict[str, Any]]:

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/routes/api/v2/task_collection.py

@@ -13,7 +13,6 @@ from fastapi import UploadFile
 from pydantic import BaseModel
 from pydantic import model_validator
 from pydantic import ValidationError
-from sqlmodel import select
 
 from .....config import get_settings
 from .....logger import reset_logger_handlers
@@ -31,6 +30,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions_task_lifecycle import get_package_version_from_pypi
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
+from ._aux_functions_tasks import _verify_non_duplication_group_path
 from ._aux_functions_tasks import _verify_non_duplication_user_constraint
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupActivityV2
@@ -291,18 +291,10 @@ async def collect_tasks_pip(
         version=task_group_attrs["version"],
         db=db,
     )
-
-    # Verify that task-group path is unique
-    stm = select(TaskGroupV2).where(TaskGroupV2.path == task_group_path)
-    res = await db.execute(stm)
-    for conflicting_task_group in res.scalars().all():
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Another task-group already has path={task_group_path}.\n"
-                f"{conflicting_task_group=}"
-            ),
-        )
+    await _verify_non_duplication_group_path(
+        path=task_group_attrs["path"],
+        db=db,
+    )
 
     # On-disk checks
 

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/routes/api/v2/task_collection_pixi.py

@@ -9,7 +9,6 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from fastapi import UploadFile
-from sqlmodel import select
 
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
@@ -22,6 +21,9 @@ from fractal_server.app.routes.api.v2._aux_functions_tasks import (
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _verify_non_duplication_group_constraint,
 )
+from fractal_server.app.routes.api.v2._aux_functions_tasks import (
+    _verify_non_duplication_group_path,
+)
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _verify_non_duplication_user_constraint,
 )
@@ -156,18 +158,10 @@ async def collect_task_pixi(
         version=task_group_attrs["version"],
         db=db,
     )
-
-    # NOTE: to be removed with issue #2634
-    stm = select(TaskGroupV2).where(TaskGroupV2.path == task_group_path)
-    res = await db.execute(stm)
-    for conflicting_task_group in res.scalars().all():
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Another task-group already has path={task_group_path}.\n"
-                f"{conflicting_task_group=}"
-            ),
-        )
+    await _verify_non_duplication_group_path(
+        path=task_group_attrs["path"],
+        db=db,
+    )
 
     if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
         if Path(task_group_path).exists():

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/routes/api/v2/task_group.py

@@ -12,6 +12,7 @@ from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
 
+from ._aux_functions_task_lifecycle import check_no_ongoing_activity
 from ._aux_functions_tasks import _get_task_group_full_access
 from ._aux_functions_tasks import _get_task_group_read_access
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
@@ -216,6 +217,8 @@ async def delete_task_group(
         db=db,
     )
 
+    await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
+
     stm = select(WorkflowTaskV2).where(
         WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
     )

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -48,6 +48,8 @@ class _SlurmConfigSet(BaseModel):
         constraint:
         gres:
         time:
+        exclude:
+        nodelist:
         account:
         extra_lines:
     """
@@ -59,6 +61,8 @@ class _SlurmConfigSet(BaseModel):
     mem: int | str | None = None
     constraint: str | None = None
     gres: str | None = None
+    exclude: str | None = None
+    nodelist: str | None = None
     time: str | None = None
     account: str | None = None
     extra_lines: list[str] | None = None
@@ -227,6 +231,8 @@ class SlurmConfig(BaseModel):
         account: Corresponds to SLURM option.
         gpus: Corresponds to SLURM option.
         time: Corresponds to SLURM option (WARNING: not fully supported).
+        nodelist: Corresponds to SLURM option.
+        exclude: Corresponds to SLURM option.
         prefix: Prefix of configuration lines in SLURM submission scripts.
         shebang_line: Shebang line for SLURM submission scripts.
         extra_lines: Additional lines to include in SLURM submission scripts.
@@ -268,6 +274,8 @@ class SlurmConfig(BaseModel):
     gpus: str | None = None
     time: str | None = None
     account: str | None = None
+    nodelist: str | None = None
+    exclude: str | None = None
 
     # Free-field attribute for extra lines to be added to the SLURM job
     # preamble
@@ -361,6 +369,8 @@ class SlurmConfig(BaseModel):
             "gpus",
             "time",
             "account",
+            "exclude",
+            "nodelist",
         ]:
             value = getattr(self, key)
             if value is not None:

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -182,33 +182,53 @@ class BaseSlurmRunner(BaseRunner):
     def _mkdir_remote_folder(self, folder: str) -> None:
         raise NotImplementedError("Implement in child class.")
 
-    def _submit_single_sbatch(
+    def _enrich_slurm_config(
         self,
-        *,
-        base_command: str,
-        slurm_job: SlurmJob,
         slurm_config: SlurmConfig,
-    ) -> str:
-        logger.debug("[_submit_single_sbatch] START")
+    ) -> SlurmConfig:
+        """
+        Return an enriched `SlurmConfig` object
 
-        # Include SLURM account in `slurm_config`. Note: we make this change
-        # here, rather than exposing a new argument of `get_slurm_config`,
-        # because it's a backend-specific argument while `get_slurm_config` has
-        # a generic interface.
+        Include `self.account` and `self.common_script_lines` into a
+        `SlurmConfig` object. Extracting this logic into an independent
+        class method is useful to fix issue #2659 (which was due to
+        performing this same operation multiple times rather than once).
+
+        Args:
+            slurm_config: The original `SlurmConfig` object.
+
+        Returns:
+            A new, up-to-date, `SlurmConfig` object.
+        """
+
+        new_slurm_config = slurm_config.model_copy()
+
+        # Include SLURM account in `slurm_config`.
         if self.slurm_account is not None:
-            slurm_config.account = self.slurm_account
+            new_slurm_config.account = self.slurm_account
 
         # Include common_script_lines in extra_lines
         if len(self.common_script_lines) > 0:
             logger.debug(
                 f"Add {self.common_script_lines} to "
-                f"{slurm_config.extra_lines=}."
+                f"{new_slurm_config.extra_lines=}."
             )
-            current_extra_lines = slurm_config.extra_lines or []
-            slurm_config.extra_lines = (
+            current_extra_lines = new_slurm_config.extra_lines or []
+            new_slurm_config.extra_lines = (
                 current_extra_lines + self.common_script_lines
             )
 
+        return new_slurm_config
+
+    def _submit_single_sbatch(
+        self,
+        *,
+        base_command: str,
+        slurm_job: SlurmJob,
+        slurm_config: SlurmConfig,
+    ) -> str:
+        logger.debug("[_submit_single_sbatch] START")
+
         for task in slurm_job.tasks:
             # Write input file
             if self.slurm_runner_type == "ssh":
@@ -508,6 +528,9 @@ class BaseSlurmRunner(BaseRunner):
         user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
+
+        config = self._enrich_slurm_config(config)
+
         try:
             workdir_local = task_files.wftask_subfolder_local
             workdir_remote = task_files.wftask_subfolder_remote
@@ -649,6 +672,8 @@ class BaseSlurmRunner(BaseRunner):
         input images, while for compound tasks these can differ.
         """
 
+        config = self._enrich_slurm_config(config)
+
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
         try:
             if self.is_shutdown():

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -125,7 +125,14 @@ def get_slurm_config_internal(
             )
             logger.error(error_msg)
             raise SlurmConfigError(error_msg)
-        for key in ["time", "gres", "gpus", "constraint"]:
+        for key in [
+            "time",
+            "gres",
+            "gpus",
+            "constraint",
+            "nodelist",
+            "exclude",
+        ]:
             value = wftask_meta.get(key, None)
             if value is not None:
                 slurm_dict[key] = value

{fractal_server-2.15.0a4 → fractal_server-2.15.1}/fractal_server/config.py

@@ -65,8 +65,36 @@ class MailSettings(BaseModel):
 
 
 class PixiSettings(BaseModel):
-    default_version: str
+    """
+    Configuration for Pixi task collection
+
+    See https://pixi.sh/latest/reference/cli/pixi/install/#config-options for
+    `pixi install` concurrency options.
+    See https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
+    for `tokio` configuration.
+
+    versions:
+        Available `pixi` versions and their `PIXI_HOME` folders.
+    default_version:
+        Default `pixi` version to use for task collection - must be one
+        of `versions` keys.
+    PIXI_CONCURRENT_SOLVES:
+        Value of `--concurrent-solves` for `pixi install`.
+    PIXI_CONCURRENT_DOWNLOADS:
+        Value of `--concurrent-downloads for `pixi install`.
+    TOKIO_WORKER_THREADS:
+        From tokio docs, "The core threads are where all asynchronous code
+        runs, and Tokio will by default spawn one for each CPU core. You can
+        use the environment variable TOKIO_WORKER_THREADS to override the
+        default value."
+    """
+
     versions: DictStrStr
+    default_version: str
+
+    PIXI_CONCURRENT_SOLVES: int = 4
+    PIXI_CONCURRENT_DOWNLOADS: int = 4
+    TOKIO_WORKER_THREADS: int = 2
 
     @model_validator(mode="after")
     def check_pixi_settings(self):