fractal-server 2.13.0__py3-none-any.whl → 2.14.0a0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.13.0"
+__VERSION__ = "2.14.0a0"

fractal_server/app/history/__init__.py

@@ -0,0 +1,4 @@
+from .image_updates import update_all_images  # noqa: F401
+from .image_updates import update_single_image  # noqa
+from .image_updates import update_single_image_logfile  # noqa
+from .status_enum import HistoryItemImageStatus  # noqa: F401

fractal_server/app/history/image_updates.py

@@ -0,0 +1,142 @@
+from typing import Optional
+
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.history.status_enum import HistoryItemImageStatus
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
+
+
+def _update_single_image_status(
+    *,
+    zarr_url: str,
+    workflowtask_id: int,
+    dataset_id: int,
+    status: HistoryItemImageStatus,
+    db: Session,
+    commit: bool = True,
+    logfile: Optional[str] = None,
+) -> None:
+    image_status = db.get(
+        ImageStatus,
+        (
+            zarr_url,
+            workflowtask_id,
+            dataset_id,
+        ),
+    )
+    if image_status is None:
+        raise RuntimeError("This should have not happened")
+    image_status.status = status
+    if logfile is not None:
+        image_status.logfile = logfile
+    db.add(image_status)
+    if commit:
+        db.commit()
+
+
+def update_single_image(
+    *,
+    history_item_id: int,
+    zarr_url: str,
+    status: HistoryItemImageStatus,
+) -> None:
+
+    logger.debug(
+        f"[update_single_image] {history_item_id=}, {status=}, {zarr_url=}"
+    )
+
+    # Note: thanks to `with_for_update`, a lock is acquired and kept
+    # until `db.commit()`
+    with next(get_sync_db()) as db:
+        stm = (
+            select(HistoryItemV2)
+            .where(HistoryItemV2.id == history_item_id)
+            .with_for_update(nowait=False)
+        )
+        history_item = db.execute(stm).scalar_one()
+        history_item.images[zarr_url] = status
+        flag_modified(history_item, "images")
+        db.commit()
+
+        _update_single_image_status(
+            zarr_url=zarr_url,
+            dataset_id=history_item.dataset_id,
+            workflowtask_id=history_item.workflowtask_id,
+            commit=True,
+            status=status,
+            db=db,
+        )
+
+
+def update_single_image_logfile(
+    *,
+    history_item_id: int,
+    zarr_url: str,
+    logfile: str,
+) -> None:
+
+    logger.debug(
+        f"[update_single_image_logfile] {history_item_id=}, {logfile=}, {zarr_url=}"
+    )
+
+    with next(get_sync_db()) as db:
+        history_item = db.get(HistoryItemV2, history_item_id)
+        image_status = db.get(
+            ImageStatus,
+            (
+                zarr_url,
+                history_item.workflowtask_id,
+                history_item.dataset_id,
+            ),
+        )
+        if image_status is None:
+            raise RuntimeError("This should have not happened")
+        image_status.logfile = logfile
+        db.merge(image_status)
+        db.commit()
+
+
+def update_all_images(
+    *,
+    history_item_id: int,
+    status: HistoryItemImageStatus,
+    logfile: Optional[str] = None,
+) -> None:
+
+    logger.debug(f"[update_all_images] {history_item_id=}, {status=}")
+
+    # Note: thanks to `with_for_update`, a lock is acquired and kept
+    # until `db.commit()`
+    stm = (
+        select(HistoryItemV2)
+        .where(HistoryItemV2.id == history_item_id)
+        .with_for_update(nowait=False)
+    )
+    with next(get_sync_db()) as db:
+        history_item = db.execute(stm).scalar_one()
+        new_images = {
+            zarr_url: status for zarr_url in history_item.images.keys()
+        }
+        history_item.images = new_images
+        flag_modified(history_item, "images")
+        db.commit()
+
+        # FIXME: Make this a bulk edit, if possible
+        for ind, zarr_url in enumerate(history_item.images.keys()):
+            _update_single_image_status(
+                zarr_url=zarr_url,
+                dataset_id=history_item.dataset_id,
+                workflowtask_id=history_item.workflowtask_id,
+                commit=False,
+                status=status,
+                logfile=logfile,
+                db=db,
+            )
+        db.commit()

fractal_server/app/history/status_enum.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class HistoryItemImageStatus(str, Enum):
+    """
+    Available image-status values within a `HistoryItemV2`
+
+    Attributes:
+        SUBMITTED:
+        DONE:
+        FAILED:
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"

fractal_server/app/models/v2/__init__.py

@@ -2,7 +2,11 @@
 v2 `models` module
 """
 from ..linkuserproject import LinkUserProjectV2
+from .accounting import AccountingRecord
+from .accounting import AccountingRecordSlurm
 from .dataset import DatasetV2
+from .history import HistoryItemV2
+from .history import ImageStatus
 from .job import JobV2
 from .project import ProjectV2
 from .task import TaskV2
@@ -12,6 +16,8 @@ from .workflow import WorkflowV2
 from .workflowtask import WorkflowTaskV2
 
 __all__ = [
+    "AccountingRecord",
+    "AccountingRecordSlurm",
     "LinkUserProjectV2",
     "DatasetV2",
     "JobV2",
@@ -19,6 +25,8 @@ __all__ = [
     "TaskGroupV2",
     "TaskGroupActivityV2",
     "TaskV2",
-    "WorkflowTaskV2",
     "WorkflowV2",
+    "WorkflowTaskV2",
+    "HistoryItemV2",
+    "ImageStatus",
 ]

fractal_server/app/models/v2/accounting.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy import Integer
+from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+
+
+class AccountingRecord(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    user_id: int = Field(foreign_key="user_oauth.id", nullable=False)
+    timestamp: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    num_tasks: int
+    num_new_images: int
+
+
+class AccountingRecordSlurm(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    user_id: int = Field(foreign_key="user_oauth.id", nullable=False)
+    timestamp: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    slurm_job_ids: list[int] = Field(
+        default_factory=list,
+        sa_column=Column(ARRAY(Integer)),
+    )

fractal_server/app/models/v2/history.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+from typing import Any
+from typing import Optional
+
+from pydantic import ConfigDict
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+
+
+class HistoryItemV2(SQLModel, table=True):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    dataset_id: int = Field(foreign_key="datasetv2.id")
+    workflowtask_id: Optional[int] = Field(
+        foreign_key="workflowtaskv2.id",
+        default=None,
+    )
+    timestamp_started: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(
+            DateTime(timezone=True),
+            nullable=False,
+        ),
+    )
+    workflowtask_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False)
+    )
+    task_group_dump: dict[str, Any] = Field(
+        sa_column=Column(JSONB, nullable=False)
+    )
+    parameters_hash: str
+    num_available_images: int
+    num_current_images: int
+    images: dict[str, str] = Field(sa_column=Column(JSONB, nullable=False))
+
+
+class ImageStatus(SQLModel, table=True):
+
+    zarr_url: str = Field(primary_key=True)
+    workflowtask_id: int = Field(
+        primary_key=True, foreign_key="workflowtaskv2.id"
+    )
+    dataset_id: int = Field(primary_key=True, foreign_key="datasetv2.id")
+
+    parameters_hash: str
+    status: str
+    logfile: str

fractal_server/app/routes/admin/v2/__init__.py

@@ -3,6 +3,8 @@
 """
 from fastapi import APIRouter
 
+from .accounting import router as accounting_router
+from .impersonate import router as impersonate_router
 from .job import router as job_router
 from .project import router as project_router
 from .task import router as task_router
@@ -11,6 +13,7 @@ from .task_group_lifecycle import router as task_group_lifecycle_router
 
 router_admin_v2 = APIRouter()
 
+router_admin_v2.include_router(accounting_router, prefix="/accounting")
 router_admin_v2.include_router(job_router, prefix="/job")
 router_admin_v2.include_router(project_router, prefix="/project")
 router_admin_v2.include_router(task_router, prefix="/task")
@@ -18,3 +21,4 @@ router_admin_v2.include_router(task_group_router, prefix="/task-group")
 router_admin_v2.include_router(
     task_group_lifecycle_router, prefix="/task-group"
 )
+router_admin_v2.include_router(impersonate_router, prefix="/impersonate")

fractal_server/app/routes/admin/v2/accounting.py

@@ -0,0 +1,108 @@
+from itertools import chain
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Query
+from fastapi import status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from pydantic.types import AwareDatetime
+from sqlmodel import func
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import AccountingRecord
+from fractal_server.app.models.v2 import AccountingRecordSlurm
+from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.schemas.v2 import AccountingRecordRead
+
+
+class AccountingQuery(BaseModel):
+    user_id: Optional[int] = None
+    timestamp_min: Optional[AwareDatetime] = None
+    timestamp_max: Optional[AwareDatetime] = None
+
+
+class AccountingPage(BaseModel):
+    total_count: int
+    page_size: int
+    current_page: int
+    records: list[AccountingRecordRead]
+
+
+router = APIRouter()
+
+
+@router.post("/", response_model=AccountingPage)
+async def query_accounting(
+    query: AccountingQuery,
+    # pagination
+    page: int = Query(default=1, ge=1),
+    page_size: Optional[int] = Query(default=None, ge=1),
+    # dependencies
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> AccountingPage:
+
+    if page_size is None and page > 1:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(f"Invalid pagination parameters: {page=}, {page_size=}."),
+        )
+
+    stm = select(AccountingRecord).order_by(AccountingRecord.id)
+    stm_count = select(func.count(AccountingRecord.id))
+    if query.user_id is not None:
+        stm = stm.where(AccountingRecord.user_id == query.user_id)
+        stm_count = stm_count.where(AccountingRecord.user_id == query.user_id)
+    if query.timestamp_min is not None:
+        stm = stm.where(AccountingRecord.timestamp >= query.timestamp_min)
+        stm_count = stm_count.where(
+            AccountingRecord.timestamp >= query.timestamp_min
+        )
+    if query.timestamp_max is not None:
+        stm = stm.where(AccountingRecord.timestamp <= query.timestamp_max)
+        stm_count = stm_count.where(
+            AccountingRecord.timestamp <= query.timestamp_max
+        )
+    if page_size is not None:
+        stm = stm.offset((page - 1) * page_size).limit(page_size)
+
+    res = await db.execute(stm)
+    records = res.scalars().all()
+    res_total_count = await db.execute(stm_count)
+    total_count = res_total_count.scalar()
+
+    actual_page_size = page_size or len(records)
+    return AccountingPage(
+        total_count=total_count,
+        page_size=actual_page_size,
+        current_page=page,
+        records=[record.model_dump() for record in records],
+    )
+
+
+@router.post("/slurm/")
+async def query_accounting_slurm(
+    query: AccountingQuery,
+    # dependencies
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+
+    stm = select(AccountingRecordSlurm.slurm_job_ids)
+    if query.user_id is not None:
+        stm = stm.where(AccountingRecordSlurm.user_id == query.user_id)
+    if query.timestamp_min is not None:
+        stm = stm.where(AccountingRecordSlurm.timestamp >= query.timestamp_min)
+    if query.timestamp_max is not None:
+        stm = stm.where(AccountingRecordSlurm.timestamp <= query.timestamp_max)
+
+    res = await db.execute(stm)
+    nested_slurm_job_ids = res.scalars().all()
+    aggregated_slurm_job_ids = list(chain(*nested_slurm_job_ids))
+    return JSONResponse(content=aggregated_slurm_job_ids, status_code=200)

fractal_server/app/routes/admin/v2/impersonate.py

@@ -0,0 +1,35 @@
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+from fastapi_users.authentication import JWTStrategy
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.auth._aux_auth import _user_or_404
+from fractal_server.config import get_settings
+from fractal_server.syringe import Inject
+
+router = APIRouter()
+
+
+@router.get("/{user_id}/")
+async def impersonate_user(
+    user_id: int,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> JSONResponse:
+    user = await _user_or_404(user_id, db)
+
+    settings = Inject(get_settings)
+    jwt_strategy = JWTStrategy(
+        secret=settings.JWT_SECRET_KEY,  # type: ignore
+        lifetime_seconds=7200,  # 2 hours
+    )
+    token = await jwt_strategy.write_token(user)
+
+    return JSONResponse(
+        content={"access_token": token, "token_type": "bearer"},
+        status_code=200,
+    )

fractal_server/app/routes/admin/v2/job.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from pathlib import Path
 from typing import Optional
 
@@ -8,6 +7,7 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
+from pydantic.types import AwareDatetime
 from sqlmodel import select
 
 from fractal_server.app.db import AsyncSession
@@ -16,7 +16,6 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.routes.auth import current_active_superuser
-from fractal_server.app.routes.aux import _raise_if_naive_datetime
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
 from fractal_server.app.runner.filenames import WORKFLOW_LOG_FILENAME
@@ -37,10 +36,10 @@ async def view_job(
     dataset_id: Optional[int] = None,
     workflow_id: Optional[int] = None,
     status: Optional[JobStatusTypeV2] = None,
-    start_timestamp_min: Optional[datetime] = None,
-    start_timestamp_max: Optional[datetime] = None,
-    end_timestamp_min: Optional[datetime] = None,
-    end_timestamp_max: Optional[datetime] = None,
+    start_timestamp_min: Optional[AwareDatetime] = None,
+    start_timestamp_max: Optional[AwareDatetime] = None,
+    end_timestamp_min: Optional[AwareDatetime] = None,
+    end_timestamp_max: Optional[AwareDatetime] = None,
     log: bool = True,
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
@@ -67,13 +66,6 @@ async def view_job(
             `job.log` is set to `None`.
     """
 
-    _raise_if_naive_datetime(
-        start_timestamp_min,
-        start_timestamp_max,
-        end_timestamp_min,
-        end_timestamp_max,
-    )
-
     stm = select(JobV2)
 
     if id is not None:

fractal_server/app/routes/admin/v2/task_group.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from typing import Optional
 
 from fastapi import APIRouter
@@ -6,6 +5,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from pydantic.types import AwareDatetime
 from sqlalchemy.sql.operators import is_
 from sqlalchemy.sql.operators import is_not
 from sqlmodel import select
@@ -20,7 +20,6 @@ from fractal_server.app.routes.auth import current_active_superuser
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
-from fractal_server.app.routes.aux import _raise_if_naive_datetime
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
@@ -42,13 +41,11 @@ async def get_task_group_activity_list(
     pkg_name: Optional[str] = None,
     status: Optional[TaskGroupActivityStatusV2] = None,
     action: Optional[TaskGroupActivityActionV2] = None,
-    timestamp_started_min: Optional[datetime] = None,
+    timestamp_started_min: Optional[AwareDatetime] = None,
     superuser: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskGroupActivityV2Read]:
 
-    _raise_if_naive_datetime(timestamp_started_min)
-
     stm = select(TaskGroupActivityV2)
     if task_group_activity_id is not None:
         stm = stm.where(TaskGroupActivityV2.id == task_group_activity_id)
@@ -96,19 +93,14 @@ async def query_task_group_list(
     active: Optional[bool] = None,
     pkg_name: Optional[str] = None,
     origin: Optional[TaskGroupV2OriginEnum] = None,
-    timestamp_last_used_min: Optional[datetime] = None,
-    timestamp_last_used_max: Optional[datetime] = None,
+    timestamp_last_used_min: Optional[AwareDatetime] = None,
+    timestamp_last_used_max: Optional[AwareDatetime] = None,
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskGroupReadV2]:
 
     stm = select(TaskGroupV2)
 
-    _raise_if_naive_datetime(
-        timestamp_last_used_max,
-        timestamp_last_used_min,
-    )
-
     if user_group_id is not None and private is True:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,

fractal_server/app/routes/api/v2/__init__.py

@@ -4,10 +4,10 @@
 from fastapi import APIRouter
 
 from .dataset import router as dataset_router_v2
+from .history import router as history_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
-from .status import router as status_router_v2
 from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
@@ -28,6 +28,7 @@ router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
 router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
+router_api_v2.include_router(history_router_v2, tags=["V2 History"])
 
 
 settings = Inject(get_settings)
@@ -56,4 +57,3 @@ router_api_v2.include_router(
     workflow_import_router_v2, tags=["V2 Workflow Import"]
 )
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
-router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -417,3 +417,81 @@ async def clean_app_job_list_v2(
         if job.status == JobStatusTypeV2.SUBMITTED
     ]
     return submitted_job_ids
+
+
+async def _get_workflow_check_history_owner(
+    *,
+    workflow_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> list[int]:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+
+    Returns:
+        List of WorkflowTask IDs
+    """
+    workflow = await db.get(WorkflowV2, workflow_id)
+    if workflow is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Workflow not found.",
+        )
+    await _get_project_check_owner(
+        project_id=workflow.project_id,
+        user_id=user_id,
+        db=db,
+    )
+    dataset = await db.get(DatasetV2, dataset_id)
+    if dataset is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="Dataset not found.",
+        )
+    if workflow.project_id != dataset.project_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Dataset and workflow belong to different projects.",
+        )
+
+    return [wftask.id for wftask in workflow.task_list]
+
+
+async def _get_workflowtask_check_history_owner(
+    *,
+    workflowtask_id: int,
+    dataset_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> list[int]:
+    """
+    Verify user access for the history of this dataset and workflowtask.
+
+    Args:
+        dataset_id:
+        workflow_task_id:
+        user_id:
+        db:
+
+    Returns:
+        List of WorkflowTask IDs
+    """
+    workflowtask = await db.get(WorkflowTaskV2, workflowtask_id)
+    if workflowtask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="WorkflowTask not found.",
+        )
+    await _get_workflow_check_history_owner(
+        workflow_id=workflowtask.workflow_id,
+        dataset_id=dataset_id,
+        user_id=user_id,
+        db=db,
+    )