fractal-server 2.14.0a2__py3-none-any.whl → 2.14.0a4__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a2"
+__VERSION__ = "2.14.0a4"

fractal_server/__main__.py

@@ -123,7 +123,9 @@ def set_db(skip_init_data: bool = False):
     asyncio.run(
         _create_first_user(
             email=settings.FRACTAL_DEFAULT_ADMIN_EMAIL,
-            password=settings.FRACTAL_DEFAULT_ADMIN_PASSWORD,
+            password=(
+                settings.FRACTAL_DEFAULT_ADMIN_PASSWORD.get_secret_value()
+            ),
             username=settings.FRACTAL_DEFAULT_ADMIN_USERNAME,
             is_superuser=True,
             is_verified=True,

fractal_server/app/history/__init__.py

@@ -1,4 +1,4 @@
-from .image_updates import update_all_images  # noqa: F401
-from .image_updates import update_single_image  # noqa
-from .image_updates import update_single_image_logfile  # noqa
-from .status_enum import HistoryItemImageStatus  # noqa: F401
+# from .image_updates import update_all_images  # noqa: F401
+# from .image_updates import update_single_image  # noqa
+# from .image_updates import update_single_image_logfile  # noqa
+# from .status_enum import XXXStatus  # noqa: F401

fractal_server/app/history/image_updates.py

@@ -1,142 +1,124 @@
-from typing import Optional
-
-from sqlalchemy.orm import Session
-from sqlalchemy.orm.attributes import flag_modified
-from sqlmodel import select
-
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.history.status_enum import HistoryItemImageStatus
-from fractal_server.app.models.v2 import HistoryItemV2
-from fractal_server.app.models.v2 import ImageStatus
-from fractal_server.logger import set_logger
-
-logger = set_logger(__name__)
-
-
-def _update_single_image_status(
-    *,
-    zarr_url: str,
-    workflowtask_id: int,
-    dataset_id: int,
-    status: HistoryItemImageStatus,
-    db: Session,
-    commit: bool = True,
-    logfile: Optional[str] = None,
-) -> None:
-    image_status = db.get(
-        ImageStatus,
-        (
-            zarr_url,
-            workflowtask_id,
-            dataset_id,
-        ),
-    )
-    if image_status is None:
-        raise RuntimeError("This should have not happened")
-    image_status.status = status
-    if logfile is not None:
-        image_status.logfile = logfile
-    db.add(image_status)
-    if commit:
-        db.commit()
-
-
-def update_single_image(
-    *,
-    history_item_id: int,
-    zarr_url: str,
-    status: HistoryItemImageStatus,
-) -> None:
-
-    logger.debug(
-        f"[update_single_image] {history_item_id=}, {status=}, {zarr_url=}"
-    )
-
-    # Note: thanks to `with_for_update`, a lock is acquired and kept
-    # until `db.commit()`
-    with next(get_sync_db()) as db:
-        stm = (
-            select(HistoryItemV2)
-            .where(HistoryItemV2.id == history_item_id)
-            .with_for_update(nowait=False)
-        )
-        history_item = db.execute(stm).scalar_one()
-        history_item.images[zarr_url] = status
-        flag_modified(history_item, "images")
-        db.commit()
-
-        _update_single_image_status(
-            zarr_url=zarr_url,
-            dataset_id=history_item.dataset_id,
-            workflowtask_id=history_item.workflowtask_id,
-            commit=True,
-            status=status,
-            db=db,
-        )
-
-
-def update_single_image_logfile(
-    *,
-    history_item_id: int,
-    zarr_url: str,
-    logfile: str,
-) -> None:
-
-    logger.debug(
-        f"[update_single_image_logfile] {history_item_id=}, {logfile=}, {zarr_url=}"
-    )
-
-    with next(get_sync_db()) as db:
-        history_item = db.get(HistoryItemV2, history_item_id)
-        image_status = db.get(
-            ImageStatus,
-            (
-                zarr_url,
-                history_item.workflowtask_id,
-                history_item.dataset_id,
-            ),
-        )
-        if image_status is None:
-            raise RuntimeError("This should have not happened")
-        image_status.logfile = logfile
-        db.merge(image_status)
-        db.commit()
-
-
-def update_all_images(
-    *,
-    history_item_id: int,
-    status: HistoryItemImageStatus,
-    logfile: Optional[str] = None,
-) -> None:
-
-    logger.debug(f"[update_all_images] {history_item_id=}, {status=}")
-
-    # Note: thanks to `with_for_update`, a lock is acquired and kept
-    # until `db.commit()`
-    stm = (
-        select(HistoryItemV2)
-        .where(HistoryItemV2.id == history_item_id)
-        .with_for_update(nowait=False)
-    )
-    with next(get_sync_db()) as db:
-        history_item = db.execute(stm).scalar_one()
-        new_images = {
-            zarr_url: status for zarr_url in history_item.images.keys()
-        }
-        history_item.images = new_images
-        flag_modified(history_item, "images")
-        db.commit()
-
-        # FIXME: Make this a bulk edit, if possible
-        for ind, zarr_url in enumerate(history_item.images.keys()):
-            _update_single_image_status(
-                zarr_url=zarr_url,
-                dataset_id=history_item.dataset_id,
-                workflowtask_id=history_item.workflowtask_id,
-                commit=False,
-                status=status,
-                logfile=logfile,
-                db=db,
-            )
-        db.commit()
+# from typing import Optional
+# from sqlalchemy.orm import Session
+# from sqlalchemy.orm.attributes import flag_modified
+# from sqlmodel import select
+# from fractal_server.app.db import get_sync_db
+# from fractal_server.app.history.status_enum import HistoryItemImageStatus
+# from fractal_server.app.models.v2 import HistoryItemV2
+# from fractal_server.app.models.v2 import ImageStatus
+# from fractal_server.logger import set_logger
+# logger = set_logger(__name__)
+# def _update_single_image_status(
+#     *,
+#     zarr_url: str,
+#     workflowtask_id: int,
+#     dataset_id: int,
+#     status: HistoryItemImageStatus,
+#     db: Session,
+#     commit: bool = True,
+#     logfile: Optional[str] = None,
+# ) -> None:
+#     image_status = db.get(
+#         ImageStatus,
+#         (
+#             zarr_url,
+#             workflowtask_id,
+#             dataset_id,
+#         ),
+#     )
+#     if image_status is None:
+#         raise RuntimeError("This should have not happened")
+#     image_status.status = status
+#     if logfile is not None:
+#         image_status.logfile = logfile
+#     db.add(image_status)
+#     if commit:
+#         db.commit()
+# def update_single_image(
+#     *,
+#     history_item_id: int,
+#     zarr_url: str,
+#     status: HistoryItemImageStatus,
+# ) -> None:
+#     logger.debug(
+#         f"[update_single_image] {history_item_id=}, {status=}, {zarr_url=}"
+#     )
+#     # Note: thanks to `with_for_update`, a lock is acquired and kept
+#     # until `db.commit()`
+#     with next(get_sync_db()) as db:
+#         stm = (
+#             select(HistoryItemV2)
+#             .where(HistoryItemV2.id == history_item_id)
+#             .with_for_update(nowait=False)
+#         )
+#         history_item = db.execute(stm).scalar_one()
+#         history_item.images[zarr_url] = status
+#         flag_modified(history_item, "images")
+#         db.commit()
+#         _update_single_image_status(
+#             zarr_url=zarr_url,
+#             dataset_id=history_item.dataset_id,
+#             workflowtask_id=history_item.workflowtask_id,
+#             commit=True,
+#             status=status,
+#             db=db,
+#         )
+# def update_single_image_logfile(
+#     *,
+#     history_item_id: int,
+#     zarr_url: str,
+#     logfile: str,
+# ) -> None:
+#     logger.debug(
+#         "[update_single_image_logfile] "
+#         f"{history_item_id=}, {logfile=}, {zarr_url=}"
+#     )
+#     with next(get_sync_db()) as db:
+#         history_item = db.get(HistoryItemV2, history_item_id)
+#         image_status = db.get(
+#             ImageStatus,
+#             (
+#                 zarr_url,
+#                 history_item.workflowtask_id,
+#                 history_item.dataset_id,
+#             ),
+#         )
+#         if image_status is None:
+#             raise RuntimeError("This should have not happened")
+#         image_status.logfile = logfile
+#         db.merge(image_status)
+#         db.commit()
+# def update_all_images(
+#     *,
+#     history_item_id: int,
+#     status: HistoryItemImageStatus,
+#     logfile: Optional[str] = None,
+# ) -> None:
+#     logger.debug(f"[update_all_images] {history_item_id=}, {status=}")
+#     # Note: thanks to `with_for_update`, a lock is acquired and kept
+#     # until `db.commit()`
+#     stm = (
+#         select(HistoryItemV2)
+#         .where(HistoryItemV2.id == history_item_id)
+#         .with_for_update(nowait=False)
+#     )
+#     with next(get_sync_db()) as db:
+#         history_item = db.execute(stm).scalar_one()
+#         new_images = {
+#             zarr_url: status for zarr_url in history_item.images.keys()
+#         }
+#         history_item.images = new_images
+#         flag_modified(history_item, "images")
+#         db.commit()
+#         # FIXME: Make this a bulk edit, if possible
+#         for ind, zarr_url in enumerate(history_item.images.keys()):
+#             _update_single_image_status(
+#                 zarr_url=zarr_url,
+#                 dataset_id=history_item.dataset_id,
+#                 workflowtask_id=history_item.workflowtask_id,
+#                 commit=False,
+#                 status=status,
+#                 logfile=logfile,
+#                 db=db,
+#             )
+#         db.commit()

fractal_server/app/history/status_enum.py

@@ -1,9 +1,9 @@
 from enum import Enum
 
 
-class HistoryItemImageStatus(str, Enum):
+class XXXStatus(str, Enum):
     """
-    Available image-status values within a `HistoryItemV2`
+    Available status for images
 
     Attributes:
         SUBMITTED:

fractal_server/app/models/v2/__init__.py

@@ -5,8 +5,9 @@ from ..linkuserproject import LinkUserProjectV2
 from .accounting import AccountingRecord
 from .accounting import AccountingRecordSlurm
 from .dataset import DatasetV2
-from .history import HistoryItemV2
-from .history import ImageStatus
+from .history import HistoryImageCache
+from .history import HistoryRun
+from .history import HistoryUnit
 from .job import JobV2
 from .project import ProjectV2
 from .task import TaskV2
@@ -27,6 +28,7 @@ __all__ = [
     "TaskV2",
     "WorkflowV2",
     "WorkflowTaskV2",
-    "HistoryItemV2",
-    "ImageStatus",
+    "HistoryRun",
+    "HistoryUnit",
+    "HistoryImageCache",
 ]

fractal_server/app/models/v2/history.py

@@ -4,6 +4,8 @@ from typing import Optional
 
 from pydantic import ConfigDict
 from sqlalchemy import Column
+from sqlalchemy import String
+from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.types import DateTime
 from sqlmodel import Field
@@ -12,42 +14,64 @@ from sqlmodel import SQLModel
 from ....utils import get_timestamp
 
 
-class HistoryItemV2(SQLModel, table=True):
+class HistoryRun(SQLModel, table=True):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     id: Optional[int] = Field(default=None, primary_key=True)
-    dataset_id: int = Field(foreign_key="datasetv2.id")
+    dataset_id: int = Field(
+        foreign_key="datasetv2.id",
+        ondelete="CASCADE",
+    )
     workflowtask_id: Optional[int] = Field(
         foreign_key="workflowtaskv2.id",
         default=None,
+        ondelete="SET NULL",
     )
-    timestamp_started: datetime = Field(
-        default_factory=get_timestamp,
-        sa_column=Column(
-            DateTime(timezone=True),
-            nullable=False,
-        ),
-    )
+
     workflowtask_dump: dict[str, Any] = Field(
-        sa_column=Column(JSONB, nullable=False)
+        sa_column=Column(JSONB, nullable=False),
     )
     task_group_dump: dict[str, Any] = Field(
-        sa_column=Column(JSONB, nullable=False)
+        sa_column=Column(JSONB, nullable=False),
     )
-    parameters_hash: str
+
+    timestamp_started: datetime = Field(
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+        default_factory=get_timestamp,
+    )
+    status: str
     num_available_images: int
-    num_current_images: int
-    images: dict[str, str] = Field(sa_column=Column(JSONB, nullable=False))
 
 
-class ImageStatus(SQLModel, table=True):
+class HistoryUnit(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
+    history_run_id: int = Field(
+        foreign_key="historyrun.id",
+        ondelete="CASCADE",
+    )
+
+    logfile: Optional[str]
+    status: str
+    zarr_urls: list[str] = Field(
+        sa_column=Column(ARRAY(String)),
+        default_factory=list,
+    )
+
 
+class HistoryImageCache(SQLModel, table=True):
     zarr_url: str = Field(primary_key=True)
+    dataset_id: int = Field(
+        primary_key=True,
+        foreign_key="datasetv2.id",
+        ondelete="CASCADE",
+    )
     workflowtask_id: int = Field(
-        primary_key=True, foreign_key="workflowtaskv2.id"
+        primary_key=True,
+        foreign_key="workflowtaskv2.id",
+        ondelete="CASCADE",
     )
-    dataset_id: int = Field(primary_key=True, foreign_key="datasetv2.id")
 
-    parameters_hash: str
-    status: str
-    logfile: Optional[str]
+    latest_history_unit_id: int = Field(
+        foreign_key="historyunit.id",
+        ondelete="CASCADE",
+    )

fractal_server/app/routes/admin/v2/task.py

@@ -68,7 +68,7 @@ async def query_tasks(
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskV2Info]:
     """
-    Query `TaskV2` table and get informations about related items
+    Query `TaskV2` table and get information about related items
     (WorkflowV2s and ProjectV2s)
 
     Args:

fractal_server/app/routes/api/__init__.py

@@ -25,4 +25,4 @@ async def alive():
 @router_api.get("/settings/")
 async def view_settings(user: UserOAuth = Depends(current_active_superuser)):
     settings = Inject(get_settings)
-    return settings.get_sanitized()
+    return settings.model_dump()

fractal_server/app/routes/api/v2/__init__.py

@@ -8,6 +8,7 @@ from .history import router as history_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
+from .status_legacy import router as status_legacy_router_v2
 from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
@@ -29,6 +30,9 @@ router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
 router_api_v2.include_router(submit_job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(history_router_v2, tags=["V2 History"])
+router_api_v2.include_router(
+    status_legacy_router_v2, tags=["V2 Status Legacy"]
+)
 
 
 settings = Inject(get_settings)

fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from fastapi import HTTPException
+from fastapi import status
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models.v2 import HistoryUnit
+
+
+async def get_history_unit_or_404(
+    *, history_unit_id: int, db: AsyncSession
+) -> HistoryUnit:
+    """
+    Get an existing HistoryUnit  or raise a 404.
+
+    Arguments:
+        history_unit_id: The `HistoryUnit` id
+        db: An asynchronous db session
+    """
+    history_unit = await db.get(HistoryUnit, history_unit_id)
+    if history_unit is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"HistoryUnit {history_unit_id} not found",
+        )
+    return history_unit
+
+
+def read_log_file(
+    *,
+    logfile: str | None,
+    wftask: WorkflowTaskV2,
+    dataset_id: int,
+):
+    if logfile is None or not Path(logfile).exists():
+        return (
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available."
+        )
+
+    try:
+        with open(logfile, "r") as f:
+            return f.read()
+    except Exception as e:
+        return (
+            f"Error while retrieving logs for task '{wftask.task.name}' "
+            f"in dataset {dataset_id}. Original error: {str(e)}."
+        )

fractal_server/app/routes/api/v2/dataset.py

@@ -5,14 +5,11 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
-from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import DatasetV2
-from ....models.v2 import HistoryItemV2
-from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....schemas.v2 import DatasetCreateV2
@@ -223,15 +220,6 @@ async def delete_dataset(
     for job in jobs:
         job.dataset_id = None
 
-    # Cascade operations: delete history items and image status which are in
-    # relationship with the current dataset
-
-    stm = delete(HistoryItemV2).where(HistoryItemV2.dataset_id == dataset_id)
-    await db.execute(stm)
-
-    stm = delete(ImageStatus).where(ImageStatus.dataset_id == dataset_id)
-    await db.execute(stm)
-
     # Delete dataset
     await db.delete(dataset)
     await db.commit()