fractal-server 2.13.1__py3-none-any.whl → 2.14.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.13.1"
+__VERSION__ = "2.14.0"

fractal_server/__main__.py

@@ -123,7 +123,9 @@ def set_db(skip_init_data: bool = False):
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
-            password=settings.FRACTAL_DEFAULT_ADMIN_PASSWORD,
+            password=(
+                settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()
+            ),
             username=settings.FRACTAL_DEFAULT_ADMIN_USERNAME,
             is_superuser=True,
             is_verified=True,

fractal_server/app/models/linkusergroup.py

@@ -13,8 +13,12 @@ class LinkUserGroup(SQLModel, table=True):
     Crossing table between User and UserGroup
     """
 
-    group_id: int = Field(foreign_key="usergroup.id", primary_key=True)
-    user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)
+    group_id: int = Field(
+        foreign_key="usergroup.id", primary_key=True, ondelete="CASCADE"
+    )
+    user_id: int = Field(
+        foreign_key="user_oauth.id", primary_key=True, ondelete="CASCADE"
+    )
 
     timestamp_created: datetime = Field(
         default_factory=get_timestamp,

fractal_server/app/models/v2/__init__.py

@@ -5,6 +5,9 @@ from ..linkuserproject import LinkUserProjectV2
 from .accounting import AccountingRecord
 from .accounting import AccountingRecordSlurm
 from .dataset import DatasetV2
+from .history import HistoryImageCache
+from .history import HistoryRun
+from .history import HistoryUnit
 from .job import JobV2
 from .project import ProjectV2
 from .task import TaskV2
@@ -23,6 +26,9 @@ __all__ = [
     "TaskGroupV2",
     "TaskGroupActivityV2",
     "TaskV2",
-    "WorkflowTaskV2",
     "WorkflowV2",
+    "WorkflowTaskV2",
+    "HistoryRun",
+    "HistoryUnit",
+    "HistoryImageCache",
 ]

fractal_server/app/models/v2/dataset.py

@@ -11,7 +11,6 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
-from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -20,7 +19,7 @@ class DatasetV2(SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
     name: str
 
-    project_id: int = Field(foreign_key="projectv2.id")
+    project_id: int = Field(foreign_key="projectv2.id", ondelete="CASCADE")
     project: "ProjectV2" = Relationship(  # noqa: F821
         sa_relationship_kwargs=dict(lazy="selectin"),
     )
@@ -34,20 +33,11 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
 
-    # New in V2
-
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-    attribute_filters: AttributeFiltersType = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-
     @property
     def image_zarr_urls(self) -> list[str]:
         return [image["zarr_url"] for image in self.images]

fractal_server/app/models/v2/history.py

@@ -0,0 +1,78 @@
+from datetime import datetime
+from typing import Any
+from typing import Optional
+
+from pydantic import ConfigDict
+from sqlalchemy import Column
+from sqlalchemy import String
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+
+
+class HistoryRun(SQLModel, table=True):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    dataset_id: int = Field(
+        foreign_key="datasetv2.id",
+        ondelete="CASCADE",
+    )
+    workflowtask_id: Optional[int] = Field(
+        foreign_key="workflowtaskv2.id",
+        default=None,
+        ondelete="SET NULL",
+    )
+    job_id: int = Field(foreign_key="jobv2.id")
+
+    workflowtask_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False),
+    )
+    task_group_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False),
+    )
+
+    timestamp_started: datetime = Field(
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+        default_factory=get_timestamp,
+    )
+    status: str
+    num_available_images: int
+
+
+class HistoryUnit(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    history_run_id: int = Field(
+        foreign_key="historyrun.id",
+        ondelete="CASCADE",
+    )
+
+    logfile: str
+    status: str
+    zarr_urls: list[str] = Field(
+        sa_column=Column(ARRAY(String)),
+        default_factory=list,
+    )
+
+
+class HistoryImageCache(SQLModel, table=True):
+    zarr_url: str = Field(primary_key=True)
+    dataset_id: int = Field(
+        primary_key=True,
+        foreign_key="datasetv2.id",
+        ondelete="CASCADE",
+    )
+    workflowtask_id: int = Field(
+        primary_key=True,
+        foreign_key="workflowtaskv2.id",
+        ondelete="CASCADE",
+    )
+
+    latest_history_unit_id: int = Field(
+        foreign_key="historyunit.id",
+        ondelete="CASCADE",
+    )

fractal_server/app/models/v2/job.py

@@ -18,11 +18,15 @@ class JobV2(SQLModel, table=True):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     id: Optional[int] = Field(default=None, primary_key=True)
-    project_id: Optional[int] = Field(foreign_key="projectv2.id", default=None)
+    project_id: Optional[int] = Field(
+        foreign_key="projectv2.id", default=None, ondelete="SET NULL"
+    )
     workflow_id: Optional[int] = Field(
-        foreign_key="workflowv2.id", default=None
+        foreign_key="workflowv2.id", default=None, ondelete="SET NULL"
+    )
+    dataset_id: Optional[int] = Field(
+        foreign_key="datasetv2.id", default=None, ondelete="SET NULL"
     )
-    dataset_id: Optional[int] = Field(foreign_key="datasetv2.id", default=None)
 
     user_email: str = Field(nullable=False)
     slurm_account: Optional[str] = None
@@ -56,3 +60,6 @@ class JobV2(SQLModel, table=True):
     attribute_filters: AttributeFiltersType = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

fractal_server/app/models/v2/task_group.py

@@ -23,7 +23,7 @@ class TaskGroupV2(SQLModel, table=True):
 
     user_id: int = Field(foreign_key="user_oauth.id")
     user_group_id: Optional[int] = Field(
-        foreign_key="usergroup.id", default=None
+        foreign_key="usergroup.id", default=None, ondelete="SET NULL"
     )
 
     origin: str
@@ -100,7 +100,7 @@ class TaskGroupActivityV2(SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
     user_id: int = Field(foreign_key="user_oauth.id")
     taskgroupv2_id: Optional[int] = Field(
-        default=None, foreign_key="taskgroupv2.id"
+        default=None, foreign_key="taskgroupv2.id", ondelete="SET NULL"
     )
     timestamp_started: datetime = Field(
         default_factory=get_timestamp,

fractal_server/app/models/v2/workflow.py

@@ -16,7 +16,7 @@ class WorkflowV2(SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
     name: str
-    project_id: int = Field(foreign_key="projectv2.id")
+    project_id: int = Field(foreign_key="projectv2.id", ondelete="CASCADE")
     project: "ProjectV2" = Relationship(  # noqa: F821
         sa_relationship_kwargs=dict(lazy="selectin"),
     )

fractal_server/app/models/v2/workflowtask.py

@@ -16,7 +16,7 @@ class WorkflowTaskV2(SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
 
-    workflow_id: int = Field(foreign_key="workflowv2.id")
+    workflow_id: int = Field(foreign_key="workflowv2.id", ondelete="CASCADE")
     order: Optional[int] = None
     meta_parallel: Optional[dict[str, Any]] = Field(
         sa_column=Column(JSON), default=None

fractal_server/app/routes/admin/v2/accounting.py

@@ -3,9 +3,6 @@ from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
-from fastapi import HTTPException
-from fastapi import Query
-from fastapi import status
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
 from pydantic.types import AwareDatetime
@@ -18,6 +15,9 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import AccountingRecordSlurm
 from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.pagination import get_pagination_params
+from fractal_server.app.routes.pagination import PaginationRequest
+from fractal_server.app.routes.pagination import PaginationResponse
 from fractal_server.app.schemas.v2 import AccountingRecordRead
 
 
@@ -27,32 +27,19 @@ class AccountingQuery(BaseModel):
     timestamp_max: Optional[AwareDatetime] = None
 
 
-class AccountingPage(BaseModel):
-    total_count: int
-    page_size: int
-    current_page: int
-    records: list[AccountingRecordRead]
-
-
 router = APIRouter()
 
 
-@router.post("/", response_model=AccountingPage)
+@router.post("/", response_model=PaginationResponse[AccountingRecordRead])
 async def query_accounting(
     query: AccountingQuery,
-    # pagination
-    page: int = Query(default=1, ge=1),
-    page_size: Optional[int] = Query(default=None, ge=1),
-    # dependencies
+    # Dependencies
+    pagination: PaginationRequest = Depends(get_pagination_params),
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> AccountingPage:
-
-    if page_size is None and page > 1:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
-        )
+) -> PaginationResponse[AccountingRecordRead]:
+    page = pagination.page
+    page_size = pagination.page_size
 
     stm = select(AccountingRecord).order_by(AccountingRecord.id)
     stm_count = select(func.count(AccountingRecord.id))
@@ -69,20 +56,23 @@ async def query_accounting(
         stm_count = stm_count.where(
             AccountingRecord.timestamp <= query.timestamp_max
         )
+
+    res_total_count = await db.execute(stm_count)
+    total_count = res_total_count.scalar()
+
     if page_size is not None:
         stm = stm.offset((page - 1) * page_size).limit(page_size)
+    else:
+        page_size = total_count
 
     res = await db.execute(stm)
     records = res.scalars().all()
-    res_total_count = await db.execute(stm_count)
-    total_count = res_total_count.scalar()
 
-    actual_page_size = page_size or len(records)
-    return AccountingPage(
+    return PaginationResponse[AccountingRecordRead](
         total_count=total_count,
-        page_size=actual_page_size,
+        page_size=page_size,
         current_page=page,
-        records=[record.model_dump() for record in records],
+        items=[record.model_dump() for record in records],
     )
 
 

fractal_server/app/routes/admin/v2/task.py

@@ -68,7 +68,7 @@ async def query_tasks(
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskV2Info]:
     """
-    Query `TaskV2` table and get informations about related items
+    Query `TaskV2` table and get information about related items
     (WorkflowV2s and ProjectV2s)
 
     Args:

fractal_server/app/routes/admin/v2/task_group.py

@@ -192,23 +192,6 @@ async def delete_task_group(
             detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
         )
 
-    # Cascade operations: set foreign-keys to null for TaskGroupActivityV2
-    # which are in relationship with the current TaskGroupV2
-    logger.debug("Start of cascade operations on TaskGroupActivityV2.")
-    stm = select(TaskGroupActivityV2).where(
-        TaskGroupActivityV2.taskgroupv2_id == task_group_id
-    )
-    res = await db.execute(stm)
-    task_group_activity_list = res.scalars().all()
-    for task_group_activity in task_group_activity_list:
-        logger.debug(
-            f"Setting TaskGroupActivityV2[{task_group_activity.id}]"
-            ".taskgroupv2_id to None."
-        )
-        task_group_activity.taskgroupv2_id = None
-        db.add(task_group_activity)
-    logger.debug("End of cascade operations on TaskGroupActivityV2.")
-
     await db.delete(task_group)
     await db.commit()
 

fractal_server/app/routes/api/__init__.py

@@ -25,4 +25,4 @@ async def alive():
 @router_api.get("/settings/")
 async def view_settings(user: UserOAuth = Depends(current_active_superuser)):
     settings = Inject(get_settings)
-    return settings.get_sanitized()
+    return settings.model_dump()

fractal_server/app/routes/api/v2/__init__.py

@@ -4,16 +4,18 @@
 from fastapi import APIRouter
 
 from .dataset import router as dataset_router_v2
+from .history import router as history_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
-from .status import router as status_router_v2
+from .status_legacy import router as status_legacy_router_v2
 from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
 from .task_group import router as task_group_router_v2
 from .task_group_lifecycle import router as task_group_lifecycle_router_v2
+from .verify_image_types import router as verify_image_types_router
 from .workflow import router as workflow_router_v2
 from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
@@ -24,10 +26,15 @@ from fractal_server.syringe import Inject
 router_api_v2 = APIRouter()
 
 router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
+router_api_v2.include_router(verify_image_types_router, tags=["V2 Job"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
 router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
+router_api_v2.include_router(history_router_v2, tags=["V2 History"])
+router_api_v2.include_router(
+    status_legacy_router_v2, tags=["V2 Status Legacy"]
+)
 
 
 settings = Inject(get_settings)
@@ -56,4 +63,3 @@ router_api_v2.include_router(
     workflow_import_router_v2, tags=["V2 Workflow Import"]
 )
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
-router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -417,3 +417,69 @@ async def clean_app_job_list_v2(
         if job.status == JobStatusTypeV2.SUBMITTED
     ]
     return submitted_job_ids
+
+
+async def _get_dataset_or_404(
+    *,
+    dataset_id: int,
+    db: AsyncSession,
+) -> DatasetV2:
+    """
+    Get a dataset or raise 404.
+
+    Args:
+        dataset_id:
+        db:
+    """
+    ds = await db.get(DatasetV2, dataset_id)
+    if ds is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Dataset {dataset_id} not found.",
+        )
+    else:
+        return ds
+
+
+async def _get_workflow_or_404(
+    *,
+    workflow_id: int,
+    db: AsyncSession,
+) -> WorkflowV2:
+    """
+    Get a workflow or raise 404.
+
+    Args:
+        workflow_id:
+        db:
+    """
+    wf = await db.get(WorkflowV2, workflow_id)
+    if wf is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Workflow {workflow_id} not found.",
+        )
+    else:
+        return wf
+
+
+async def _get_workflowtask_or_404(
+    *,
+    workflowtask_id: int,
+    db: AsyncSession,
+) -> WorkflowTaskV2:
+    """
+    Get a workflow task or raise 404.
+
+    Args:
+        workflowtask_id:
+        db:
+    """
+    wftask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if wftask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"WorkflowTask {workflowtask_id} not found.",
+        )
+    else:
+        return wftask

fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -0,0 +1,166 @@
+from pathlib import Path
+from typing import Literal
+
+from fastapi import HTTPException
+from fastapi import status
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import HistoryRun
+from fractal_server.app.models.v2 import HistoryUnit
+from fractal_server.app.models.v2 import WorkflowV2
+from fractal_server.app.routes.api.v2._aux_functions import _get_dataset_or_404
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_project_check_owner,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflow_or_404,
+)
+from fractal_server.app.routes.api.v2._aux_functions import (
+    _get_workflowtask_or_404,
+)
+from fractal_server.logger import set_logger
+
+
+logger = set_logger(__name__)
+
+
+async def get_history_unit_or_404(
+    *, history_unit_id: int, db: AsyncSession
+) -> HistoryUnit:
+    """
+    Get an existing HistoryUnit  or raise a 404.
+
+    Arguments:
+        history_unit_id: The `HistoryUnit` id
+        db: An asynchronous db session
+    """
+    history_unit = await db.get(HistoryUnit, history_unit_id)
+    if history_unit is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryUnit {history_unit_id} not found",
+        )
+    return history_unit
+
+
+async def get_history_run_or_404(
+    *, history_run_id: int, db: AsyncSession
+) -> HistoryRun:
+    """
+    Get an existing HistoryRun  or raise a 404.
+
+    Arguments:
+        history_run_id:
+        db:
+    """
+    history_run = await db.get(HistoryRun, history_run_id)
+    if history_run is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryRun {history_run_id} not found",
+        )
+    return history_run
+
+
+def read_log_file(
+    *,
+    logfile: str | None,
+    wftask: WorkflowTaskV2,
+    dataset_id: int,
+):
+    if logfile is None or not Path(logfile).exists():
+        logger.debug(
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available ({logfile=})."
+        )
+        return (
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available."
+        )
+
+    try:
+        with open(logfile, "r") as f:
+            return f.read()
+    except Exception as e:
+        return (
+            f"Error while retrieving logs for task '{wftask.task.name}' "
+            f"in dataset {dataset_id}. Original error: {str(e)}."
+        )
+
+
+async def _verify_workflow_and_dataset_access(
+    *,
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> dict[Literal["dataset", "workflow"], DatasetV2 | WorkflowV2]:
+    """
+    Verify user access to a dataset/workflow pair.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    await _get_project_check_owner(
+        project_id=project_id,
+        user_id=user_id,
+        db=db,
+    )
+    workflow = await _get_workflow_or_404(
+        workflow_id=workflow_id,
+        db=db,
+    )
+    if workflow.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Workflow does not belong to expected project.",
+        )
+    dataset = await _get_dataset_or_404(
+        dataset_id=dataset_id,
+        db=db,
+    )
+    if dataset.project_id != project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Dataset does not belong to expected project.",
+        )
+
+    return dict(dataset=dataset, workflow=workflow)
+
+
+async def get_wftask_check_owner(
+    *,
+    project_id: int,
+    dataset_id: int,
+    workflowtask_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> WorkflowTaskV2:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        project_id:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+    """
+    wftask = await _get_workflowtask_or_404(
+        workflowtask_id=workflowtask_id,
+        db=db,
+    )
+    await _verify_workflow_and_dataset_access(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        workflow_id=wftask.workflow_id,
+        user_id=user_id,
+        db=db,
+    )
+    return wftask