fractal-server 2.14.16__py3-none-any.whl → 2.15.0__py3-none-any.whl

fractal_server/app/routes/api/v2/task_group_lifecycle.py

@@ -2,7 +2,6 @@ from fastapi import APIRouter
 from fastapi import BackgroundTasks
 from fastapi import Depends
 from fastapi import HTTPException
-from fastapi import Request
 from fastapi import Response
 from fastapi import status
 
@@ -25,9 +24,13 @@ from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.local import deactivate_local
+from fractal_server.tasks.v2.local import deactivate_local_pixi
 from fractal_server.tasks.v2.local import reactivate_local
+from fractal_server.tasks.v2.local import reactivate_local_pixi
 from fractal_server.tasks.v2.ssh import deactivate_ssh
+from fractal_server.tasks.v2.ssh import deactivate_ssh_pixi
 from fractal_server.tasks.v2.ssh import reactivate_ssh
+from fractal_server.tasks.v2.ssh import reactivate_ssh_pixi
 from fractal_server.utils import get_timestamp
 
 router = APIRouter()
@@ -44,7 +47,6 @@ async def deactivate_task_group(
     task_group_id: int,
     background_tasks: BackgroundTasks,
     response: Response,
-    request: Request,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupReadV2:
@@ -125,9 +127,12 @@ async def deactivate_task_group(
             host=user_settings.ssh_host,
             key_path=user_settings.ssh_private_key_path,
         )
-
+        if task_group.origin == TaskGroupV2OriginEnum.PIXI:
+            deactivate_function = deactivate_ssh_pixi
+        else:
+            deactivate_function = deactivate_ssh
         background_tasks.add_task(
-            deactivate_ssh,
+            deactivate_function,
             task_group_id=task_group.id,
             task_group_activity_id=task_group_activity.id,
             ssh_config=ssh_config,
@@ -135,8 +140,12 @@ async def deactivate_task_group(
         )
 
     else:
+        if task_group.origin == TaskGroupV2OriginEnum.PIXI:
+            deactivate_function = deactivate_local_pixi
+        else:
+            deactivate_function = deactivate_local
         background_tasks.add_task(
-            deactivate_local,
+            deactivate_function,
             task_group_id=task_group.id,
             task_group_activity_id=task_group_activity.id,
         )
@@ -157,7 +166,6 @@ async def reactivate_task_group(
     task_group_id: int,
     background_tasks: BackgroundTasks,
     response: Response,
-    request: Request,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> TaskGroupReadV2:
@@ -210,12 +218,12 @@ async def reactivate_task_group(
         response.status_code = status.HTTP_202_ACCEPTED
         return task_group_activity
 
-    if task_group.pip_freeze is None:
+    if task_group.env_info is None:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
                 "Cannot reactivate a task group with "
-                f"{task_group.pip_freeze=}."
+                f"{task_group.env_info=}."
             ),
         )
 
@@ -247,8 +255,12 @@ async def reactivate_task_group(
             key_path=user_settings.ssh_private_key_path,
         )
 
+        if task_group.origin == TaskGroupV2OriginEnum.PIXI:
+            reactivate_function = reactivate_ssh_pixi
+        else:
+            reactivate_function = reactivate_ssh
         background_tasks.add_task(
-            reactivate_ssh,
+            reactivate_function,
             task_group_id=task_group.id,
             task_group_activity_id=task_group_activity.id,
             ssh_config=ssh_config,
@@ -256,8 +268,12 @@ async def reactivate_task_group(
         )
 
     else:
+        if task_group.origin == TaskGroupV2OriginEnum.PIXI:
+            reactivate_function = reactivate_local_pixi
+        else:
+            reactivate_function = reactivate_local
         background_tasks.add_task(
-            reactivate_local,
+            reactivate_function,
             task_group_id=task_group.id,
             task_group_activity_id=task_group_activity.id,
         )

fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -48,6 +48,8 @@ class _SlurmConfigSet(BaseModel):
         constraint:
         gres:
         time:
+        exclude:
+        nodelist:
         account:
         extra_lines:
     """
@@ -59,6 +61,8 @@ class _SlurmConfigSet(BaseModel):
     mem: int | str | None = None
     constraint: str | None = None
     gres: str | None = None
+    exclude: str | None = None
+    nodelist: str | None = None
     time: str | None = None
     account: str | None = None
     extra_lines: list[str] | None = None
@@ -227,6 +231,8 @@ class SlurmConfig(BaseModel):
         account: Corresponds to SLURM option.
         gpus: Corresponds to SLURM option.
         time: Corresponds to SLURM option (WARNING: not fully supported).
+        nodelist: Corresponds to SLURM option.
+        exclude: Corresponds to SLURM option.
         prefix: Prefix of configuration lines in SLURM submission scripts.
         shebang_line: Shebang line for SLURM submission scripts.
         extra_lines: Additional lines to include in SLURM submission scripts.
@@ -268,6 +274,8 @@ class SlurmConfig(BaseModel):
     gpus: str | None = None
     time: str | None = None
     account: str | None = None
+    nodelist: str | None = None
+    exclude: str | None = None
 
     # Free-field attribute for extra lines to be added to the SLURM job
     # preamble
@@ -361,6 +369,8 @@ class SlurmConfig(BaseModel):
             "gpus",
             "time",
             "account",
+            "exclude",
+            "nodelist",
         ]:
             value = getattr(self, key)
             if value is not None:

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -182,33 +182,53 @@ class BaseSlurmRunner(BaseRunner):
     def _mkdir_remote_folder(self, folder: str) -> None:
         raise NotImplementedError("Implement in child class.")
 
-    def _submit_single_sbatch(
+    def _enrich_slurm_config(
         self,
-        *,
-        base_command: str,
-        slurm_job: SlurmJob,
         slurm_config: SlurmConfig,
-    ) -> str:
-        logger.debug("[_submit_single_sbatch] START")
+    ) -> SlurmConfig:
+        """
+        Return an enriched `SlurmConfig` object
 
-        # Include SLURM account in `slurm_config`. Note: we make this change
-        # here, rather than exposing a new argument of `get_slurm_config`,
-        # because it's a backend-specific argument while `get_slurm_config` has
-        # a generic interface.
+        Include `self.account` and `self.common_script_lines` into a
+        `SlurmConfig` object. Extracting this logic into an independent
+        class method is useful to fix issue #2659 (which was due to
+        performing this same operation multiple times rather than once).
+
+        Args:
+            slurm_config: The original `SlurmConfig` object.
+
+        Returns:
+            A new, up-to-date, `SlurmConfig` object.
+        """
+
+        new_slurm_config = slurm_config.model_copy()
+
+        # Include SLURM account in `slurm_config`.
         if self.slurm_account is not None:
-            slurm_config.account = self.slurm_account
+            new_slurm_config.account = self.slurm_account
 
         # Include common_script_lines in extra_lines
         if len(self.common_script_lines) > 0:
             logger.debug(
                 f"Add {self.common_script_lines} to "
-                f"{slurm_config.extra_lines=}."
+                f"{new_slurm_config.extra_lines=}."
             )
-            current_extra_lines = slurm_config.extra_lines or []
-            slurm_config.extra_lines = (
+            current_extra_lines = new_slurm_config.extra_lines or []
+            new_slurm_config.extra_lines = (
                 current_extra_lines + self.common_script_lines
             )
 
+        return new_slurm_config
+
+    def _submit_single_sbatch(
+        self,
+        *,
+        base_command: str,
+        slurm_job: SlurmJob,
+        slurm_config: SlurmConfig,
+    ) -> str:
+        logger.debug("[_submit_single_sbatch] START")
+
         for task in slurm_job.tasks:
             # Write input file
             if self.slurm_runner_type == "ssh":
@@ -508,6 +528,9 @@ class BaseSlurmRunner(BaseRunner):
         user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
+
+        config = self._enrich_slurm_config(config)
+
         try:
             workdir_local = task_files.wftask_subfolder_local
             workdir_remote = task_files.wftask_subfolder_remote
@@ -649,6 +672,8 @@ class BaseSlurmRunner(BaseRunner):
         input images, while for compound tasks these can differ.
         """
 
+        config = self._enrich_slurm_config(config)
+
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
         try:
             if self.is_shutdown():

fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -125,7 +125,14 @@ def get_slurm_config_internal(
             )
             logger.error(error_msg)
             raise SlurmConfigError(error_msg)
-        for key in ["time", "gres", "gpus", "constraint"]:
+        for key in [
+            "time",
+            "gres",
+            "gpus",
+            "constraint",
+            "nodelist",
+            "exclude",
+        ]:
             value = wftask_meta.get(key, None)
             if value is not None:
                 slurm_dict[key] = value

fractal_server/app/schemas/v2/__init__.py

@@ -33,9 +33,9 @@ from .task import TaskImportV2Legacy  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskType  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
+from .task_collection import FractalUploadedFile  # noqa F401
 from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401
-from .task_collection import WheelFile  # noqa F401
 from .task_group import TaskGroupActivityActionV2  # noqa F401
 from .task_group import TaskGroupActivityStatusV2  # noqa F401
 from .task_group import TaskGroupActivityV2Read  # noqa F401

fractal_server/app/schemas/v2/dumps.py

@@ -86,4 +86,4 @@ class TaskGroupDumpV2(BaseModel):
 
     path: str | None = None
     venv_path: str | None = None
-    wheel_path: str | None = None
+    archive_path: str | None = None

fractal_server/app/schemas/v2/task_collection.py

@@ -12,7 +12,7 @@ from fractal_server.types import DictStrStr
 from fractal_server.types import NonEmptyStr
 
 
-class WheelFile(BaseModel):
+class FractalUploadedFile(BaseModel):
     """
     Model for data sent from the endpoint to the background task.
     """

fractal_server/app/schemas/v2/task_group.py

@@ -16,6 +16,7 @@ from fractal_server.types import NonEmptyStr
 class TaskGroupV2OriginEnum(StrEnum):
     PYPI = "pypi"
     WHEELFILE = "wheel-file"
+    PIXI = "pixi"
     OTHER = "other"
 
 
@@ -41,11 +42,12 @@ class TaskGroupCreateV2(BaseModel):
     pkg_name: str
     version: str | None = None
     python_version: NonEmptyStr = None
+    pixi_version: NonEmptyStr = None
     path: AbsolutePathStr = None
     venv_path: AbsolutePathStr = None
-    wheel_path: AbsolutePathStr = None
+    archive_path: AbsolutePathStr = None
     pip_extras: NonEmptyStr = None
-    pip_freeze: str | None = None
+    env_info: str | None = None
     pinned_package_versions: DictStrStr = Field(default_factory=dict)
 
 
@@ -55,8 +57,8 @@ class TaskGroupCreateV2Strict(TaskGroupCreateV2):
     """
 
     path: AbsolutePathStr
-    venv_path: AbsolutePathStr
     version: NonEmptyStr
+    venv_path: AbsolutePathStr
     python_version: NonEmptyStr
 
 
@@ -71,10 +73,10 @@ class TaskGroupReadV2(BaseModel):
     pkg_name: str
     version: str | None = None
     python_version: str | None = None
+    pixi_version: str | None = None
     path: str | None = None
     venv_path: str | None = None
-    wheel_path: str | None = None
-    pip_freeze: str | None = None
+    archive_path: str | None = None
     pip_extras: str | None = None
     pinned_package_versions: dict[str, str] = Field(default_factory=dict)
 

fractal_server/config.py

@@ -11,6 +11,7 @@
 # <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
 # Institute for Biomedical Research and Pelkmans Lab from the University of
 # Zurich.
+import json
 import logging
 import shutil
 import sys
@@ -34,6 +35,7 @@ from sqlalchemy.engine import URL
 
 import fractal_server
 from fractal_server.types import AbsolutePathStr
+from fractal_server.types import DictStrStr
 
 
 class MailSettings(BaseModel):
@@ -62,6 +64,63 @@ class MailSettings(BaseModel):
     use_login: bool
 
 
+class PixiSettings(BaseModel):
+    """
+    Configuration for Pixi task collection
+
+    See https://pixi.sh/latest/reference/cli/pixi/install/#config-options for
+    `pixi install` concurrency options.
+    See https://docs.rs/tokio/latest/tokio/#cpu-bound-tasks-and-blocking-code
+    for `tokio` configuration.
+
+    versions:
+        Available `pixi` versions and their `PIXI_HOME` folders.
+    default_version:
+        Default `pixi` version to use for task collection - must be one
+        of `versions` keys.
+    PIXI_CONCURRENT_SOLVES:
+        Value of `--concurrent-solves` for `pixi install`.
+    PIXI_CONCURRENT_DOWNLOADS:
+        Value of `--concurrent-downloads for `pixi install`.
+    TOKIO_WORKER_THREADS:
+        From tokio docs, "The core threads are where all asynchronous code
+        runs, and Tokio will by default spawn one for each CPU core. You can
+        use the environment variable TOKIO_WORKER_THREADS to override the
+        default value."
+    """
+
+    versions: DictStrStr
+    default_version: str
+
+    PIXI_CONCURRENT_SOLVES: int = 4
+    PIXI_CONCURRENT_DOWNLOADS: int = 4
+    TOKIO_WORKER_THREADS: int = 2
+
+    @model_validator(mode="after")
+    def check_pixi_settings(self):
+
+        if self.default_version not in self.versions:
+            raise ValueError(
+                f"Default version '{self.default_version}' not in "
+                f"available version {list(self.versions.keys())}."
+            )
+
+        pixi_base_dir = Path(self.versions[self.default_version]).parent
+
+        for key, value in self.versions.items():
+
+            pixi_path = Path(value)
+
+            if pixi_path.parent != pixi_base_dir:
+                raise ValueError(
+                    f"{pixi_path=} is not located within the {pixi_base_dir=}."
+                )
+            if pixi_path.name != key:
+                raise ValueError(f"{pixi_path.name=} is not equal to {key=}")
+
+        return self
+
+
 class FractalConfigurationError(RuntimeError):
     pass
 
@@ -513,6 +572,17 @@ class Settings(BaseSettings):
     FRACTAL_VIEWER_AUTHORIZATION_SCHEME is set to "users-folders".
     """
 
+    FRACTAL_PIXI_CONFIG_FILE: Path | None = None
+
+    pixi: PixiSettings | None = None
+
+    @model_validator(mode="after")
+    def populate_pixi_settings(self):
+        if self.FRACTAL_PIXI_CONFIG_FILE is not None:
+            with self.FRACTAL_PIXI_CONFIG_FILE.open("r") as f:
+                self.pixi = PixiSettings(**json.load(f))
+        return self
+
     ###########################################################################
     # SMTP SERVICE
     ###########################################################################

fractal_server/migrations/versions/b1e7f7a1ff71_task_group_for_pixi.py

@@ -0,0 +1,53 @@
+"""Task group for pixi
+
+Revision ID: b1e7f7a1ff71
+Revises: 791ce783d3d8
+Create Date: 2025-05-29 16:31:17.565973
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b1e7f7a1ff71"
+down_revision = "791ce783d3d8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "pixi_version",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=True,
+            )
+        )
+        batch_op.alter_column(
+            "wheel_path",
+            nullable=True,
+            new_column_name="archive_path",
+        )
+        batch_op.alter_column(
+            "pip_freeze",
+            nullable=True,
+            new_column_name="env_info",
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "archive_path",
+            nullable=True,
+            new_column_name="wheel_path",
+        )
+        batch_op.alter_column(
+            "env_info",
+            nullable=True,
+            new_column_name="pip_freeze",
+        )
+        batch_op.drop_column("pixi_version")