fractal-server 2.17.0a11__tar.gz → 2.17.1a0__tar.gz

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fractal-server
-Version: 2.17.0a11
+Version: 2.17.1a0
 Summary: Backend component of the Fractal analytics platform
 License-Expression: BSD-3-Clause
 License-File: LICENSE

fractal_server-2.17.1a0/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.17.1a0"

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/__main__.py

@@ -7,7 +7,6 @@ from pathlib import Path
 import uvicorn
 from pydantic import ValidationError
 
-from fractal_server.app.schemas.v2 import ResourceType
 
 parser = ap.ArgumentParser(description="fractal-server commands")
 
@@ -149,6 +148,7 @@ def init_db_data(
     from fractal_server.app.models import Resource, Profile
     from fractal_server.app.schemas.v2.resource import cast_serialize_resource
     from fractal_server.app.schemas.v2.profile import cast_serialize_profile
+    from fractal_server.app.schemas.v2 import ResourceType
 
     # Create default group and user
     print()

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/app/routes/admin/v2/job.py

@@ -7,6 +7,7 @@ from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
 from pydantic.types import AwareDatetime
+from sqlalchemy import func
 from sqlmodel import select
 
 from fractal_server.app.db import AsyncSession
@@ -19,6 +20,9 @@ from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.routes.auth import current_superuser_act
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import JobReadV2
 from fractal_server.app.schemas.v2 import JobStatusTypeV2
@@ -30,7 +34,7 @@ from fractal_server.zip_tools import _zip_folder_to_byte_stream_iterator
 router = APIRouter()
 
 
-@router.get("/", response_model=list[JobReadV2])
+@router.get("/", response_model=PaginationResponse[JobReadV2])
 async def view_job(
     id: int | None = None,
     user_id: int | None = None,
@@ -43,9 +47,10 @@ async def view_job(
     end_timestamp_min: AwareDatetime | None = None,
     end_timestamp_max: AwareDatetime | None = None,
     log: bool = True,
+    pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_superuser_act),
     db: AsyncSession = Depends(get_async_db),
-) -> list[JobReadV2]:
+) -> PaginationResponse[JobReadV2]:
     """
     Query `ApplyWorkflow` table.
 
@@ -68,43 +73,74 @@ async def view_job(
             `job.log` is set to `None`.
     """
 
-    stm = select(JobV2)
+    # Assign pagination parameters
+    page = pagination.page
+    page_size = pagination.page_size
 
+    # Prepare statements
+    stm = select(JobV2).order_by(JobV2.start_timestamp.desc())
+    stm_count = select(func.count(JobV2.id))
     if id is not None:
         stm = stm.where(JobV2.id == id)
+        stm_count = stm_count.where(JobV2.id == id)
     if user_id is not None:
         stm = stm.join(ProjectV2).where(
             ProjectV2.user_list.any(UserOAuth.id == user_id)
         )
+        stm_count = stm_count.join(ProjectV2).where(
+            ProjectV2.user_list.any(UserOAuth.id == user_id)
+        )
     if project_id is not None:
         stm = stm.where(JobV2.project_id == project_id)
+        stm_count = stm_count.where(JobV2.project_id == project_id)
     if dataset_id is not None:
         stm = stm.where(JobV2.dataset_id == dataset_id)
+        stm_count = stm_count.where(JobV2.dataset_id == dataset_id)
     if workflow_id is not None:
         stm = stm.where(JobV2.workflow_id == workflow_id)
+        stm_count = stm_count.where(JobV2.workflow_id == workflow_id)
     if status is not None:
         stm = stm.where(JobV2.status == status)
+        stm_count = stm_count.where(JobV2.status == status)
     if start_timestamp_min is not None:
-        start_timestamp_min = start_timestamp_min
         stm = stm.where(JobV2.start_timestamp >= start_timestamp_min)
+        stm_count = stm_count.where(
+            JobV2.start_timestamp >= start_timestamp_min
+        )
     if start_timestamp_max is not None:
-        start_timestamp_max = start_timestamp_max
         stm = stm.where(JobV2.start_timestamp <= start_timestamp_max)
+        stm_count = stm_count.where(
+            JobV2.start_timestamp <= start_timestamp_max
+        )
     if end_timestamp_min is not None:
-        end_timestamp_min = end_timestamp_min
         stm = stm.where(JobV2.end_timestamp >= end_timestamp_min)
+        stm_count = stm_count.where(JobV2.end_timestamp >= end_timestamp_min)
     if end_timestamp_max is not None:
-        end_timestamp_max = end_timestamp_max
         stm = stm.where(JobV2.end_timestamp <= end_timestamp_max)
+        stm_count = stm_count.where(JobV2.end_timestamp <= end_timestamp_max)
 
+    # Find total number of elements
+    res_total_count = await db.execute(stm_count)
+    total_count = res_total_count.scalar()
+    if page_size is None:
+        page_size = total_count
+    else:
+        stm = stm.offset((page - 1) * page_size).limit(page_size)
+
+    # Get `page_size` rows
     res = await db.execute(stm)
     job_list = res.scalars().all()
-    await db.close()
+
     if not log:
         for job in job_list:
             setattr(job, "log", None)
 
-    return job_list
+    return PaginationResponse[JobReadV2](
+        total_count=total_count,
+        page_size=page_size,
+        current_page=page,
+        items=[job.model_dump() for job in job_list],
+    )
 
 
 @router.get("/{job_id}/", response_model=JobReadV2)

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/app/routes/admin/v2/task.py

@@ -1,7 +1,5 @@
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import HTTPException
-from fastapi import status
 from pydantic import BaseModel
 from pydantic import EmailStr
 from pydantic import Field
@@ -16,6 +14,10 @@ from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
 from fractal_server.app.routes.auth import current_superuser_act
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
+from fractal_server.app.schemas.v2.task import TaskType
 
 router = APIRouter()
 
@@ -49,75 +51,85 @@ class TaskV2Info(BaseModel):
     relationships: list[TaskV2Relationship]
 
 
-@router.get("/", response_model=list[TaskV2Info])
+@router.get("/", response_model=PaginationResponse[TaskV2Info])
 async def query_tasks(
     id: int | None = None,
     source: str | None = None,
     version: str | None = None,
     name: str | None = None,
-    max_number_of_results: int = 25,
+    task_type: TaskType | None = None,
     category: str | None = None,
     modality: str | None = None,
     author: str | None = None,
     resource_id: int | None = None,
+    pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_superuser_act),
     db: AsyncSession = Depends(get_async_db),
-) -> list[TaskV2Info]:
+) -> PaginationResponse[TaskV2Info]:
     """
-    Query `TaskV2` table and get information about related items
-    (WorkflowV2s and ProjectV2s)
-
-    Args:
-        id: If not `None`, query for matching `task.id`.
-        source: If not `None`, query for contained case insensitive
-            `task.source`.
-        version: If not `None`, query for matching `task.version`.
-        name: If not `None`, query for contained case insensitive `task.name`.
-        max_number_of_results: The maximum length of the response.
-        category:
-        modality:
-        author:
-        resource_id:
+    Query `TaskV2` and get information about related workflows and projects.
     """
 
-    stm = select(TaskV2)
+    # Assign pagination parameters
+    page = pagination.page
+    page_size = pagination.page_size
 
+    # Prepare statements
+    stm = select(TaskV2).order_by(TaskV2.id)
+    stm_count = select(func.count(TaskV2.id))
     if id is not None:
         stm = stm.where(TaskV2.id == id)
+        stm_count = stm_count.where(TaskV2.id == id)
     if source is not None:
         stm = stm.where(TaskV2.source.icontains(source))
+        stm_count = stm_count.where(TaskV2.source.icontains(source))
     if version is not None:
         stm = stm.where(TaskV2.version == version)
+        stm_count = stm_count.where(TaskV2.version == version)
     if name is not None:
         stm = stm.where(TaskV2.name.icontains(name))
+        stm_count = stm_count.where(TaskV2.name.icontains(name))
+    if task_type is not None:
+        stm = stm.where(TaskV2.type == task_type)
+        stm_count = stm_count.where(TaskV2.type == task_type)
     if category is not None:
         stm = stm.where(func.lower(TaskV2.category) == category.lower())
+        stm_count = stm_count.where(
+            func.lower(TaskV2.category) == category.lower()
+        )
     if modality is not None:
         stm = stm.where(func.lower(TaskV2.modality) == modality.lower())
+        stm_count = stm_count.where(
+            func.lower(TaskV2.modality) == modality.lower()
+        )
     if author is not None:
         stm = stm.where(TaskV2.authors.icontains(author))
+        stm_count = stm_count.where(TaskV2.authors.icontains(author))
     if resource_id is not None:
         stm = (
             stm.join(TaskGroupV2)
             .where(TaskGroupV2.id == TaskV2.taskgroupv2_id)
             .where(TaskGroupV2.resource_id == resource_id)
         )
+        stm_count = (
+            stm_count.join(TaskGroupV2)
+            .where(TaskGroupV2.id == TaskV2.taskgroupv2_id)
+            .where(TaskGroupV2.resource_id == resource_id)
+        )
+
+    # Find total number of elements
+    res_total_count = await db.execute(stm_count)
+    total_count = res_total_count.scalar()
+    if page_size is None:
+        page_size = total_count
+    else:
+        stm = stm.offset((page - 1) * page_size).limit(page_size)
 
-    stm = stm.order_by(TaskV2.id)
+    # Get `page_size` rows
     res = await db.execute(stm)
     task_list = res.scalars().all()
-    if len(task_list) > max_number_of_results:
-        await db.close()
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
-            detail=(
-                f"Too many Tasks ({len(task_list)} > {max_number_of_results})."
-                " Please add more query filters."
-            ),
-        )
 
     task_info_list = []
-
     for task in task_list:
         stm = (
             select(WorkflowV2)
@@ -147,4 +159,9 @@ async def query_tasks(
             )
         )
 
-    return task_info_list
+    return PaginationResponse[TaskV2Info](
+        total_count=total_count,
+        page_size=page_size,
+        current_page=page,
+        items=task_info_list,
+    )

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/app/schemas/v2/task_collection.py

@@ -51,9 +51,17 @@ class TaskCollectPipV2(BaseModel):
     package: NonEmptyStr | None = None
     package_version: NonEmptyStr | None = None
     package_extras: NonEmptyStr | None = None
-    python_version: Literal[
-        "3.9", "3.10", "3.11", "3.12", "3.13"
-    ] | None = None
+    python_version: (
+        Literal[
+            "3.9",
+            "3.10",
+            "3.11",
+            "3.12",
+            "3.13",
+            "3.14",
+        ]
+        | None
+    ) = None
     pinned_package_versions_pre: DictStrStr | None = None
     pinned_package_versions_post: DictStrStr | None = None
 

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/config/_database.py

@@ -1,4 +1,5 @@
 from pydantic import SecretStr
+from pydantic.types import NonNegativeInt
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 from sqlalchemy.engine import URL
@@ -27,11 +28,11 @@ class DatabaseSettings(BaseSettings):
     """
     Password to use when connecting to the PostgreSQL database.
     """
-    POSTGRES_HOST: NonEmptyStr | None = "localhost"
+    POSTGRES_HOST: NonEmptyStr = "localhost"
     """
     URL to the PostgreSQL server or path to a UNIX domain socket.
     """
-    POSTGRES_PORT: NonEmptyStr | None = "5432"
+    POSTGRES_PORT: NonNegativeInt = 5432
     """
     Port number to use when connecting to the PostgreSQL server.
     """

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/data_migrations/2_17_0.py

@@ -1,28 +1,9 @@
-"""
-
-PRELIMINARY CHECKS (TO DO WITH 2.16)
-* All users who are meant to actually use Fractal must be marked as active and verified.
-* All users who are not active and verified will still be able to log in, but they won't have access to the rest of the API.
-* All active users must have `project_dir` set, in their user settings.
-* `FRACTAL_SLURM_WORKER_PYTHON` must be included explicitly in the old env file.
-
-DATA-MIGRATION REQUIREMENTS
-* Old `.fractal_server.env`, renamed into `.fractal_server.env.old`.
-* New `.fractal_server.env` - see XXX for list of changes.
-* Old JSON file with SLURM configuration.
-* Old JSON file with pixi configuration - if applicable.
-
-
-MANUAL FIXES POST DATA MIGRATION:
-* Rename resource
-* Rename profiles - if needed
-"""
 import json
 import logging
 import sys
+from pathlib import Path
 from typing import Any
 
-from devtools import debug
 from dotenv.main import DotEnv
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
@@ -40,6 +21,7 @@ from fractal_server.app.models import UserSettings
 from fractal_server.app.schemas.v2.profile import cast_serialize_profile
 from fractal_server.app.schemas.v2.resource import cast_serialize_resource
 from fractal_server.config import get_settings
+from fractal_server.runner.config import JobRunnerConfigLocal
 from fractal_server.runner.config import JobRunnerConfigSLURM
 from fractal_server.tasks.config import TasksPixiSettings
 from fractal_server.tasks.config import TasksPythonSettings
@@ -47,8 +29,7 @@ from fractal_server.types import AbsolutePathStr
 from fractal_server.types import ListUniqueNonEmptyString
 from fractal_server.urls import normalize_url
 
-logger = logging.getLogger("fix_db")
-logger.setLevel(logging.INFO)
+logging.basicConfig(level=logging.INFO)
 
 
 class UserUpdateInfo(BaseModel):
@@ -100,6 +81,8 @@ def prepare_profile_and_user_updates() -> dict[str, ProfileUsersUpdateInfo]:
 
             # Prepare profile data and user update
             new_profile_data = dict()
+            if settings.FRACTAL_RUNNER_BACKEND == "local":
+                username = None
             if settings.FRACTAL_RUNNER_BACKEND == "slurm_sudo":
                 assert_user_setting_key(user, user_settings, ["slurm_user"])
                 username = user_settings.slurm_user
@@ -128,7 +111,6 @@ def prepare_profile_and_user_updates() -> dict[str, ProfileUsersUpdateInfo]:
                 username=username,
                 resource_type=settings.FRACTAL_RUNNER_BACKEND,
             )
-            debug(new_profile_data)
             cast_serialize_profile(new_profile_data)
 
             user_update_info = UserUpdateInfo(
@@ -139,10 +121,13 @@ def prepare_profile_and_user_updates() -> dict[str, ProfileUsersUpdateInfo]:
 
             if username in profiles_and_users.keys():
                 if profiles_and_users[username].data != new_profile_data:
-                    # FIXME
-                    debug(new_profile_data)
-                    debug(profiles_and_users[username].data)
-                    raise ValueError()
+                    error_msg = (
+                        "Profile data mismatch.\n"
+                        f"{profiles_and_users[username].data=}\n"
+                        f"{new_profile_data=}"
+                    )
+                    logging.error(error_msg)
+                    sys.exit(error_msg)
                 profiles_and_users[username].user_updates.append(
                     user_update_info
                 )
@@ -207,6 +192,19 @@ def get_JobRunnerConfigSLURM(
     return old_slurm_config
 
 
+def get_JobRunnerConfigLocal(
+    old_config: dict[str, str | None]
+) -> dict[str, Any]:
+    local_file = old_config.get("FRACTAL_LOCAL_CONFIG_FILE", None)
+    if local_file is None or not Path(local_file).exists():
+        return JobRunnerConfigLocal().model_dump()
+    else:
+        with open(local_file) as f:
+            old_local_config = json.load(f)
+        JobRunnerConfigLocal(**old_local_config)
+        return old_local_config
+
+
 def get_ssh_host() -> str:
     with next(get_sync_db()) as db:
         res = db.execute(
@@ -231,15 +229,30 @@ def prepare_resource_data(old_config: dict[str, str | None]) -> dict[str, Any]:
         name="Resource Name",
         tasks_python_config=get_TasksPythonSettings(old_config),
         tasks_pixi_config=get_TasksPixiSettings(old_config),
-        jobs_runner_config=get_JobRunnerConfigSLURM(old_config),
         tasks_local_dir=old_config["FRACTAL_TASKS_DIR"],
         jobs_local_dir=old_config["FRACTAL_RUNNER_WORKING_BASE_DIR"],
-        jobs_slurm_python_worker=old_config["FRACTAL_SLURM_WORKER_PYTHON"],
         jobs_poll_interval=int(
             old_config.get("FRACTAL_SLURM_POLL_INTERVAL", 15)
         ),
     )
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+    if settings.FRACTAL_RUNNER_BACKEND == "local":
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigLocal(
+            old_config
+        )
+    elif settings.FRACTAL_RUNNER_BACKEND == "slurm_sudo":
+        resource_data["jobs_slurm_python_worker"] = old_config[
+            "FRACTAL_SLURM_WORKER_PYTHON"
+        ]
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigSLURM(
+            old_config
+        )
+    else:
+        resource_data["jobs_slurm_python_worker"] = old_config[
+            "FRACTAL_SLURM_WORKER_PYTHON"
+        ]
+        resource_data["jobs_runner_config"] = get_JobRunnerConfigSLURM(
+            old_config
+        )
         resource_data["host"] = get_ssh_host()
 
     resource_data = cast_serialize_resource(resource_data)
@@ -248,29 +261,23 @@ def prepare_resource_data(old_config: dict[str, str | None]) -> dict[str, Any]:
 
 
 def fix_db():
-    # READ-ONLY CHECK
-
-    settings = get_settings()
-
-    # Verify that we are in a SLURM instance
-    if settings.FRACTAL_RUNNER_BACKEND == "local":
-        sys.exit(
-            "ERROR: FRACTAL_RUNNER_BACKEND='local' is not "
-            "supported for this data migration."
-        )
+    logging.info("START preliminary checks.")
 
     # Read old env file
     old_config = get_old_dotenv_variables()
 
     # Prepare resource data
+    logging.info("START prepare_resource_data")
     resource_data = prepare_resource_data(old_config)
+    logging.info("END prepare_resource_data")
 
     # Prepare profile/users data
+    logging.info("START prepare_profile_and_user_updates")
     profile_and_user_updates = prepare_profile_and_user_updates()
+    logging.info("END prepare_profile_and_user_updates")
 
-    # ---------------------------------------
-
-    # WRITES
+    logging.info("END preliminary checks.")
+    print()
 
     with next(get_sync_db()) as db:
         # Create new resource
@@ -280,7 +287,7 @@ def fix_db():
         db.refresh(resource)
         db.expunge(resource)
         resource_id = resource.id
-        debug(f"CREATED RESOURCE with {resource_id=}")
+        logging.info(f"Created resource with {resource_id=}.")
 
         # Update task groups
         res = db.execute(select(TaskGroupV2).order_by(TaskGroupV2.id))
@@ -288,6 +295,7 @@ def fix_db():
             taskgroup.resource_id = resource_id
             db.add(taskgroup)
         db.commit()
+        logging.info(f"Set {resource_id=} foreign key for all task groups.")
 
         # Update projects
         res = db.execute(select(ProjectV2).order_by(ProjectV2.id))
@@ -295,12 +303,12 @@ def fix_db():
             project.resource_id = resource_id
             db.add(project)
         db.commit()
+        logging.info(f"Set {resource_id=} foreign key for all projects.")
+        print()
 
         db.expunge_all()
 
         for _, info in profile_and_user_updates.items():
-            debug(info)
-
             # Create profile
             profile_data = info.data
             profile_data["resource_id"] = resource_id
@@ -310,6 +318,9 @@ def fix_db():
             db.refresh(profile)
             db.expunge(profile)
             profile_id = profile.id
+            logging.info(
+                f"Created profile '{profile.name}', with {profile.id=}."
+            )
 
             # Update users
             for user_update in info.user_updates:
@@ -318,4 +329,11 @@ def fix_db():
                 user.project_dir = user_update.project_dir
                 user.slurm_accounts = user_update.slurm_accounts
                 db.add(user)
+                logging.info(f"Updated {user.email} with {user.project_dir=}.")
+                logging.info(
+                    f"Associated {user.email} to profile {profile.name}."
+                )
+            print()
             db.commit()
+
+    logging.info("END - all ok.")

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/runner/v2/runner.py

@@ -192,6 +192,30 @@ def execute_tasks_v2(
             job_db.executor_error_log = None
             db.merge(job_db)
             db.commit()
+            db.expunge_all()
+
+        # Fail when running a non-converter task on an empty image list
+        if (
+            wftask.task_type
+            in [
+                TaskType.COMPOUND,
+                TaskType.PARALLEL,
+                TaskType.NON_PARALLEL,
+            ]
+            and len(filtered_images) == 0
+        ):
+            error_msg = (
+                f"Cannot run task '{task.name}' for an empty image list "
+                f"(obtained after applying {type_filters=} and "
+                f"attribute_filters={job_attribute_filters})."
+            )
+            logger.info(error_msg)
+            update_status_of_history_run(
+                history_run_id=history_run_id,
+                status=HistoryUnitStatus.FAILED,
+                db_sync=db,
+            )
+            raise JobExecutionError(error_msg)
 
         # TASK EXECUTION (V2)
         try:

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/fractal_server/tasks/config/_python.py

@@ -45,7 +45,7 @@ class TasksPythonSettings(BaseModel):
         if self.default_version not in self.versions.keys():
             raise ValueError(
                 f"The default Python version ('{self.default_version}') is "
-                f"not available versions in {list(self.versions.keys())}."
+                f"not available in {list(self.versions.keys())}."
             )
 
         return self

{fractal_server-2.17.0a11 → fractal_server-2.17.1a0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.17.0a11"
+version = "2.17.1a0"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -94,7 +94,7 @@ filterwarnings = [
 markers = ["container", "ssh", "fails_on_macos", "oauth"]
 
 [tool.bumpver]
-current_version = "2.17.0a11"
+current_version = "2.17.1a0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.17.0a11/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.17.0a11"