fractal-server 2.17.2__py3-none-any.whl → 2.18.0__py3-none-any.whl

fractal_server/main.py

@@ -1,15 +1,20 @@
 import os
+import time
 from contextlib import asynccontextmanager
+from datetime import datetime
 from itertools import chain
 
 from fastapi import FastAPI
+from starlette.types import Message
+from starlette.types import Receive
+from starlette.types import Scope
+from starlette.types import Send
 
 from fractal_server import __VERSION__
 from fractal_server.app.schemas.v2 import ResourceType
 
 from .app.routes.aux._runner import _backend_supports_shutdown
 from .app.shutdown import cleanup_after_shutdown
-from .config import get_data_settings
 from .config import get_db_settings
 from .config import get_email_settings
 from .config import get_settings
@@ -28,16 +33,14 @@ def collect_routers(app: FastAPI) -> None:
         app:
             The application to register the routers to.
     """
-    from .app.routes.admin.v2 import router_admin_v2
+    from .app.routes.admin.v2 import router_admin
     from .app.routes.api import router_api
-    from .app.routes.api.v2 import router_api_v2
+    from .app.routes.api.v2 import router_api as router_api_v2
     from .app.routes.auth.router import router_auth
 
     app.include_router(router_api, prefix="/api")
     app.include_router(router_api_v2, prefix="/api/v2")
-    app.include_router(
-        router_admin_v2, prefix="/admin/v2", tags=["V2 Admin area"]
-    )
+    app.include_router(router_admin, prefix="/admin/v2", tags=["Admin area"])
     app.include_router(router_auth, prefix="/auth", tags=["Authentication"])
 
 
@@ -54,14 +57,12 @@ def check_settings() -> None:
     settings = Inject(get_settings)
     db_settings = Inject(get_db_settings)
     email_settings = Inject(get_email_settings)
-    data_settings = Inject(get_data_settings)
     logger = set_logger("fractal_server_settings")
     logger.debug("Fractal Settings:")
     for key, value in chain(
         db_settings.model_dump().items(),
         settings.model_dump().items(),
         email_settings.model_dump().items(),
-        data_settings.model_dump().items(),
     ):
         if any(s in key.upper() for s in ["PASSWORD", "SECRET", "KEY"]):
             value = "*****"
@@ -71,7 +72,7 @@ def check_settings() -> None:
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    app.state.jobsV2 = []
+    app.state.jobs = []
     logger = set_logger("fractal_server.lifespan")
     logger.info(f"[startup] START (fractal-server {__VERSION__})")
     check_settings()
@@ -108,12 +109,12 @@ async def lifespan(app: FastAPI):
 
     logger.info(
         f"[teardown] Current worker with pid {os.getpid()} is shutting down. "
-        f"Current jobs: {app.state.jobsV2=}"
+        f"Current jobs: {app.state.jobs=}"
     )
     if _backend_supports_shutdown(settings.FRACTAL_RUNNER_BACKEND):
         try:
             await cleanup_after_shutdown(
-                jobsV2=app.state.jobsV2,
+                jobs=app.state.jobs,
                 logger_name="fractal_server.lifespan",
             )
         except Exception as e:
@@ -131,6 +132,59 @@ async def lifespan(app: FastAPI):
     reset_logger_handlers(logger)
 
 
+slow_response_logger = set_logger("slow-response")
+
+
+def _endpoint_has_background_task(method: str, path: str) -> bool:
+    has_background_task = (method == "POST") and (
+        "/job/submit/" in path
+        or "/task/collect/pi" in path  # "/pip" and "/pixi"
+        or "/task-group/" in path
+    )
+    return has_background_task
+
+
+class SlowResponseMiddleware:
+    def __init__(self, app: FastAPI, time_threshold: float):
+        self.app = app
+        self.time_threshold = time_threshold
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send):
+        if (
+            scope["type"] != "http"  # e.g. `scope["type"] == "lifespan"`
+            or _endpoint_has_background_task(scope["method"], scope["path"])
+        ):
+            await self.app(scope, receive, send)
+            return
+
+        # Mutable variable which can be updated from within `send_wrapper`
+        context = {"status_code": None}
+
+        async def send_wrapper(message: Message):
+            if message["type"] == "http.response.start":
+                context["status_code"] = message["status"]
+            await send(message)
+
+        # Measure request time
+        start_timestamp = datetime.now()
+        start_time = time.perf_counter()
+        await self.app(scope, receive, send_wrapper)
+        stop_time = time.perf_counter()
+        request_time = stop_time - start_time
+
+        # Log if process time is too high
+        if request_time > self.time_threshold:
+            end_timestamp = datetime.now()
+            slow_response_logger.warning(
+                f"{scope['method']} {scope['route'].path}"
+                f"?{scope['query_string'].decode('utf-8')}, "
+                f"{context['status_code']}, "
+                f"{request_time:.2f}, "
+                f"{start_timestamp.isoformat(timespec='milliseconds')}, "
+                f"{end_timestamp.isoformat(timespec='milliseconds')}"
+            )
+
+
 def start_application() -> FastAPI:
     """
     Create the application, initialise it and collect all available routers.
@@ -140,6 +194,13 @@ def start_application() -> FastAPI:
             The fully initialised application.
     """
     app = FastAPI(lifespan=lifespan)
+
+    settings = Inject(get_settings)
+    app.add_middleware(
+        SlowResponseMiddleware,
+        time_threshold=settings.FRACTAL_LONG_REQUEST_TIME,
+    )
+
     collect_routers(app)
     return app
 

fractal_server/migrations/versions/7910eed4cf97_user_project_dirs_and_usergroup_viewer_.py

@@ -0,0 +1,60 @@
+"""User project_dirs and UserGroup viewer paths
+
+Revision ID: 7910eed4cf97
+Revises: bc0e8b3327a7
+Create Date: 2025-11-27 16:02:51.824653
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "7910eed4cf97"
+down_revision = "bc0e8b3327a7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "project_dirs",
+                postgresql.ARRAY(sa.String()),
+                server_default="{}",
+                nullable=False,
+            )
+        )
+        batch_op.alter_column(
+            "project_dir", existing_type=sa.VARCHAR(), nullable=True
+        )
+
+    with op.batch_alter_table("usergroup", schema=None) as batch_op:
+        batch_op.drop_column("viewer_paths")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("usergroup", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "viewer_paths",
+                postgresql.JSONB(astext_type=sa.Text()),
+                server_default=sa.text("'[]'::json"),
+                autoincrement=False,
+                nullable=False,
+            )
+        )
+
+    with op.batch_alter_table("user_oauth", schema=None) as batch_op:
+        batch_op.alter_column(
+            "project_dir", existing_type=sa.VARCHAR(), nullable=False
+        )
+        batch_op.drop_column("project_dirs")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/88270f589c9b_add_prevent_new_submissions.py

@@ -0,0 +1,39 @@
+"""add_prevent_new_submissions
+
+Revision ID: 88270f589c9b
+Revises: f0702066b007
+Create Date: 2025-12-02 12:34:11.028259
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "88270f589c9b"
+down_revision = "f0702066b007"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("resource", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "prevent_new_submissions",
+                sa.BOOLEAN(),
+                server_default="false",
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("resource", schema=None) as batch_op:
+        batch_op.drop_column("prevent_new_submissions")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/bc0e8b3327a7_project_sharing.py

@@ -0,0 +1,72 @@
+"""Project sharing
+
+Revision ID: bc0e8b3327a7
+Revises: e0e717ae2f26
+Create Date: 2025-11-20 11:40:03.796112
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "bc0e8b3327a7"
+down_revision = "e0e717ae2f26"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("linkuserprojectv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "is_owner", sa.BOOLEAN(), server_default="true", nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "is_verified",
+                sa.BOOLEAN(),
+                server_default="true",
+                nullable=False,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "permissions", sa.String(), server_default="rwx", nullable=False
+            )
+        )
+        batch_op.create_index(
+            "ix_linkuserprojectv2_one_owner_per_project",
+            ["project_id"],
+            unique=True,
+            postgresql_where=sa.text("is_owner IS true"),
+        )
+
+        # ### end Alembic commands ###
+
+        # Manually add check constraints
+        batch_op.create_check_constraint(
+            "owner_is_verified", "NOT (is_owner AND NOT is_verified)"
+        )
+        batch_op.create_check_constraint(
+            "owner_full_permissions", "NOT (is_owner AND permissions <> 'rwx')"
+        )
+        batch_op.create_check_constraint(
+            "valid_permissions", "permissions IN ('r', 'rw', 'rwx')"
+        )
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("linkuserprojectv2", schema=None) as batch_op:
+        batch_op.drop_index(
+            "ix_linkuserprojectv2_one_owner_per_project",
+            postgresql_where=sa.text("is_owner IS true"),
+        )
+        batch_op.drop_column("permissions")
+        batch_op.drop_column("is_verified")
+        batch_op.drop_column("is_owner")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/f0702066b007_one_submitted_job_per_dataset.py

@@ -0,0 +1,40 @@
+"""One submitted Job per Dataset
+
+Revision ID: f0702066b007
+Revises: 7910eed4cf97
+Create Date: 2025-12-01 20:54:03.137093
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f0702066b007"
+down_revision = "7910eed4cf97"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.create_index(
+            "ix_jobv2_one_submitted_job_per_dataset",
+            ["dataset_id"],
+            unique=True,
+            postgresql_where=sa.text("status = 'submitted'"),
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_index(
+            "ix_jobv2_one_submitted_job_per_dataset",
+            postgresql_where=sa.text("status = 'submitted'"),
+        )
+
+    # ### end Alembic commands ###

fractal_server/runner/config/_slurm.py

@@ -34,6 +34,7 @@ class SlurmConfigSet(BaseModel):
         account:
         extra_lines:
         gpus:
+        shebang_line: The shell shebang to use for SLURM jobs.
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -49,6 +50,7 @@ class SlurmConfigSet(BaseModel):
     account: NonEmptyStr | None = None
     extra_lines: list[NonEmptyStr] = Field(default_factory=list)
     gpus: NonEmptyStr | None = None
+    shebang_line: str = "#!/bin/sh"
 
 
 class BatchingConfigSet(BaseModel):

fractal_server/runner/executors/slurm_common/_batching.py

@@ -1,13 +1,3 @@
-# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
-# University of Zurich
-#
-# Original authors:
-# Tommaso Comparin <tommaso.comparin@exact-lab.it>
-#
-# This file is part of Fractal and was originally developed by eXact lab S.r.l.
-# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
-# Institute for Biomedical Research and Pelkmans Lab from the University of
-# Zurich.
 """
 Submodule to determine the number of total/parallel tasks per SLURM job.
 """
@@ -20,6 +10,10 @@ logger = set_logger(__name__)
 
 
 class SlurmHeuristicsError(ValueError):
+    """
+    Error in SLURM-batching heuristics.
+    """
+
     pass
 
 

fractal_server/runner/executors/slurm_common/slurm_config.py

@@ -60,6 +60,7 @@ class SlurmConfig(BaseModel):
             Key-value pairs to be included as `export`-ed variables in SLURM
             submission script, after prepending values with the user's cache
             directory.
+
     """
 
     model_config = ConfigDict(extra="forbid")

fractal_server/runner/executors/slurm_ssh/runner.py

@@ -24,7 +24,7 @@ logger = set_logger(__name__)
 
 class SlurmSSHRunner(BaseSlurmRunner):
     """
-    Runner implementation for a computational `sudo_slurm` resource.
+    Runner implementation for a computational `slurm_ssh` resource.
     """
 
     fractal_ssh: FractalSSH

fractal_server/runner/executors/slurm_sudo/runner.py

@@ -47,7 +47,7 @@ def _subprocess_run_or_raise(
 
 class SlurmSudoRunner(BaseSlurmRunner):
     """
-    Runner implementation for a computational `sudo_slurm` resource.
+    Runner implementation for a computational `slurm_sudo` resource.
     """
 
     slurm_user: str

fractal_server/runner/v2/_local.py

@@ -14,7 +14,7 @@ from fractal_server.runner.set_start_and_last_task_index import (
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.types import AttributeFilters
 
-from .runner import execute_tasks_v2
+from .runner import execute_tasks
 
 
 def process_workflow(
@@ -59,7 +59,7 @@ def process_workflow(
         resource: Computational resource for running this job.
         profile: Computational profile for running this job.
         user_cache_dir:
-            User-writeable folder (typically a subfolder of `project_dir`).
+            User-writeable folder (typically a subfolder of `project_dirs`).
             Only relevant for `slurm_sudo` and `slurm_ssh` backends.
         fractal_ssh:
             `FractalSSH` object, only relevant for the `slurm_ssh` backend.
@@ -90,7 +90,7 @@ def process_workflow(
         resource=resource,
         profile=profile,
     ) as runner:
-        execute_tasks_v2(
+        execute_tasks(
             wf_task_list=workflow.task_list[
                 first_task_index : (last_task_index + 1)
             ],
@@ -104,4 +104,5 @@ def process_workflow(
             job_attribute_filters=job_attribute_filters,
             job_type_filters=job_type_filters,
             user_id=user_id,
+            resource_id=resource.id,
         )

fractal_server/runner/v2/_slurm_ssh.py

@@ -33,7 +33,7 @@ from fractal_server.runner.set_start_and_last_task_index import (
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.types import AttributeFilters
 
-from .runner import execute_tasks_v2
+from .runner import execute_tasks
 
 logger = set_logger(__name__)
 
@@ -80,7 +80,7 @@ def process_workflow(
         resource: Computational resource for running this job.
         profile: Computational profile for running this job.
         user_cache_dir:
-            User-writeable folder (typically a subfolder of `project_dir`).
+            User-writeable folder (typically a subfolder of `project_dirs`).
             Only relevant for `slurm_sudo` and `slurm_ssh` backends.
         fractal_ssh:
             `FractalSSH` object, only relevant for the `slurm_ssh` backend.
@@ -113,7 +113,7 @@ def process_workflow(
         common_script_lines=worker_init,
         user_cache_dir=user_cache_dir,
     ) as runner:
-        execute_tasks_v2(
+        execute_tasks(
             wf_task_list=workflow.task_list[
                 first_task_index : (last_task_index + 1)
             ],
@@ -127,4 +127,5 @@ def process_workflow(
             job_attribute_filters=job_attribute_filters,
             job_type_filters=job_type_filters,
             user_id=user_id,
+            resource_id=resource.id,
         )

fractal_server/runner/v2/_slurm_sudo.py

@@ -32,7 +32,7 @@ from fractal_server.runner.set_start_and_last_task_index import (
 from fractal_server.ssh._fabric import FractalSSH
 from fractal_server.types import AttributeFilters
 
-from .runner import execute_tasks_v2
+from .runner import execute_tasks
 
 
 def process_workflow(
@@ -77,7 +77,7 @@ def process_workflow(
         resource: Computational resource for running this job.
         profile: Computational profile for running this job.
         user_cache_dir:
-            User-writeable folder (typically a subfolder of `project_dir`).
+            User-writeable folder (typically a subfolder of `project_dirs`).
             Only relevant for `slurm_sudo` and `slurm_ssh` backends.
         fractal_ssh:
             `FractalSSH` object, only relevant for the `slurm_ssh` backend.
@@ -109,7 +109,7 @@ def process_workflow(
         user_cache_dir=user_cache_dir,
         slurm_account=slurm_account,
     ) as runner:
-        execute_tasks_v2(
+        execute_tasks(
             wf_task_list=workflow.task_list[
                 first_task_index : (last_task_index + 1)
             ],
@@ -123,4 +123,5 @@ def process_workflow(
             job_attribute_filters=job_attribute_filters,
             job_type_filters=job_type_filters,
             user_id=user_id,
+            resource_id=resource.id,
         )

fractal_server/runner/v2/runner.py

@@ -14,11 +14,12 @@ from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.models.v2 import JobV2
+from fractal_server.app.models.v2 import Resource
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
-from fractal_server.app.schemas.v2 import TaskDumpV2
-from fractal_server.app.schemas.v2 import TaskGroupDumpV2
+from fractal_server.app.schemas.v2 import TaskDump
+from fractal_server.app.schemas.v2 import TaskGroupDump
 from fractal_server.app.schemas.v2 import TaskType
 from fractal_server.images import SingleImage
 from fractal_server.images.status_tools import IMAGE_STATUS_KEY
@@ -35,9 +36,9 @@ from fractal_server.types import AttributeFilters
 from .merge_outputs import merge_outputs
 from .runner_functions import GetRunnerConfigType
 from .runner_functions import SubmissionOutcome
-from .runner_functions import run_v2_task_compound
-from .runner_functions import run_v2_task_non_parallel
-from .runner_functions import run_v2_task_parallel
+from .runner_functions import run_task_compound
+from .runner_functions import run_task_non_parallel
+from .runner_functions import run_task_parallel
 from .task_interface import TaskOutput
 
 
@@ -82,7 +83,7 @@ def get_origin_attribute_and_types(
     return updated_attributes, updated_types
 
 
-def execute_tasks_v2(
+def execute_tasks(
     *,
     wf_task_list: list[WorkflowTaskV2],
     dataset: DatasetV2,
@@ -95,6 +96,7 @@ def execute_tasks_v2(
     get_runner_config: GetRunnerConfigType,
     job_type_filters: dict[str, bool],
     job_attribute_filters: AttributeFilters,
+    resource_id: int,
 ) -> None:
     logger = get_logger(logger_name=logger_name)
 
@@ -165,10 +167,10 @@ def execute_tasks_v2(
             # Create dumps for workflowtask and taskgroup
             workflowtask_dump = dict(
                 **wftask.model_dump(exclude={"task"}),
-                task=TaskDumpV2(**wftask.task.model_dump()).model_dump(),
+                task=TaskDump(**wftask.task.model_dump()).model_dump(),
             )
             task_group = db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
-            task_group_dump = TaskGroupDumpV2(
+            task_group_dump = TaskGroupDump(
                 **task_group.model_dump()
             ).model_dump()
             # Create HistoryRun
@@ -211,20 +213,37 @@ def execute_tasks_v2(
                 f"attribute_filters={job_attribute_filters})."
             )
             logger.info(error_msg)
-            update_status_of_history_run(
-                history_run_id=history_run_id,
-                status=HistoryUnitStatus.FAILED,
-                db_sync=db,
-            )
+            with next(get_sync_db()) as db:
+                update_status_of_history_run(
+                    history_run_id=history_run_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
             raise JobExecutionError(error_msg)
 
-        # TASK EXECUTION (V2)
+        # Fail if the resource is not open for new submissions
+        with next(get_sync_db()) as db:
+            resource = db.get(Resource, resource_id)
+            if resource.prevent_new_submissions:
+                error_msg = (
+                    f"Cannot run '{task.name}', since the '{resource.name}' "
+                    "resource is not currently active."
+                )
+                logger.info(error_msg)
+                update_status_of_history_run(
+                    history_run_id=history_run_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
+                raise JobExecutionError(error_msg)
+
+        # TASK EXECUTION
         try:
             if task.type in [
                 TaskType.NON_PARALLEL,
                 TaskType.CONVERTER_NON_PARALLEL,
             ]:
-                outcomes_dict, num_tasks = run_v2_task_non_parallel(
+                outcomes_dict, num_tasks = run_task_non_parallel(
                     images=filtered_images,
                     zarr_dir=zarr_dir,
                     wftask=wftask,
@@ -239,7 +258,7 @@ def execute_tasks_v2(
                     user_id=user_id,
                 )
             elif task.type == TaskType.PARALLEL:
-                outcomes_dict, num_tasks = run_v2_task_parallel(
+                outcomes_dict, num_tasks = run_task_parallel(
                     images=filtered_images,
                     wftask=wftask,
                     task=task,
@@ -255,7 +274,7 @@ def execute_tasks_v2(
                 TaskType.COMPOUND,
                 TaskType.CONVERTER_COMPOUND,
             ]:
-                outcomes_dict, num_tasks = run_v2_task_compound(
+                outcomes_dict, num_tasks = run_task_compound(
                     images=filtered_images,
                     zarr_dir=zarr_dir,
                     wftask=wftask,