fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/routes/api/v2/dataset.py

@@ -0,0 +1,293 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import DatasetV2
+from ....models.v2 import JobV2
+from ....models.v2 import ProjectV2
+from ....schemas.v2 import DatasetCreateV2
+from ....schemas.v2 import DatasetReadV2
+from ....schemas.v2 import DatasetUpdateV2
+from ....schemas.v2.dataset import DatasetExportV2
+from ....schemas.v2.dataset import DatasetImportV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/dataset/",
+    response_model=DatasetReadV2,
+    status_code=status.HTTP_201_CREATED,
+)
+async def create_dataset(
+    project_id: int,
+    dataset: DatasetCreateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[DatasetReadV2]:
+    """
+    Add new dataset to current project
+    """
+    await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+    db_dataset = DatasetV2(project_id=project_id, **dataset.dict())
+    db.add(db_dataset)
+    await db.commit()
+    await db.refresh(db_dataset)
+    await db.close()
+
+    return db_dataset
+
+
+@router.get(
+    "/project/{project_id}/dataset/",
+    response_model=list[DatasetReadV2],
+)
+async def read_dataset_list(
+    project_id: int,
+    history: bool = True,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[list[DatasetReadV2]]:
+    """
+    Get dataset list for given project
+    """
+    # Access control
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+    # Find datasets of the current project. Note: this select/where approach
+    # has much better scaling than refreshing all elements of
+    # `project.dataset_list` - ref
+    # https://github.com/fractal-analytics-platform/fractal-server/pull/1082#issuecomment-1856676097.
+    stm = select(DatasetV2).where(DatasetV2.project_id == project.id)
+    res = await db.execute(stm)
+    dataset_list = res.scalars().all()
+    await db.close()
+    if not history:
+        for ds in dataset_list:
+            setattr(ds, "history", [])
+    return dataset_list
+
+
+@router.get(
+    "/project/{project_id}/dataset/{dataset_id}/",
+    response_model=DatasetReadV2,
+)
+async def read_dataset(
+    project_id: int,
+    dataset_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[DatasetReadV2]:
+    """
+    Get info on a dataset associated to the current project
+    """
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    dataset = output["dataset"]
+    await db.close()
+    return dataset
+
+
+@router.patch(
+    "/project/{project_id}/dataset/{dataset_id}/",
+    response_model=DatasetReadV2,
+)
+async def update_dataset(
+    project_id: int,
+    dataset_id: int,
+    dataset_update: DatasetUpdateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[DatasetReadV2]:
+    """
+    Edit a dataset associated to the current project
+    """
+
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    db_dataset = output["dataset"]
+
+    if (dataset_update.zarr_dir is not None) and (len(db_dataset.images) != 0):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot modify `zarr_dir` because the dataset has a non-empty "
+                "image list."
+            ),
+        )
+
+    for key, value in dataset_update.dict(exclude_unset=True).items():
+        setattr(db_dataset, key, value)
+
+    await db.commit()
+    await db.refresh(db_dataset)
+    await db.close()
+    return db_dataset
+
+
+@router.delete(
+    "/project/{project_id}/dataset/{dataset_id}/",
+    status_code=204,
+)
+async def delete_dataset(
+    project_id: int,
+    dataset_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete a dataset associated to the current project
+    """
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    dataset = output["dataset"]
+
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current dataset.
+    stm = _get_submitted_jobs_statement().where(JobV2.dataset_id == dataset_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    if jobs:
+        string_ids = str([job.id for job in jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot delete dataset {dataset.id} because it "
+                f"is linked to active job(s) {string_ids}."
+            ),
+        )
+
+    # Cascade operations: set foreign-keys to null for jobs which are in
+    # relationship with the current dataset
+    stm = select(JobV2).where(JobV2.dataset_id == dataset_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    for job in jobs:
+        job.dataset_id = None
+
+    # Delete dataset
+    await db.delete(dataset)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.get("/dataset/", response_model=list[DatasetReadV2])
+async def get_user_datasets(
+    history: bool = True,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[DatasetReadV2]:
+    """
+    Returns all the datasets of the current user
+    """
+    stm = select(DatasetV2)
+    stm = stm.join(ProjectV2).where(
+        ProjectV2.user_list.any(User.id == user.id)
+    )
+
+    res = await db.execute(stm)
+    dataset_list = res.scalars().all()
+    await db.close()
+    if not history:
+        for ds in dataset_list:
+            setattr(ds, "history", [])
+    return dataset_list
+
+
+@router.get(
+    "/project/{project_id}/dataset/{dataset_id}/export/",
+    response_model=DatasetExportV2,
+)
+async def export_dataset(
+    project_id: int,
+    dataset_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[DatasetExportV2]:
+    """
+    Export an existing dataset
+    """
+    dict_dataset_project = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    await db.close()
+
+    dataset = dict_dataset_project["dataset"]
+
+    return dataset
+
+
+@router.post(
+    "/project/{project_id}/dataset/import/",
+    response_model=DatasetReadV2,
+    status_code=status.HTTP_201_CREATED,
+)
+async def import_dataset(
+    project_id: int,
+    dataset: DatasetImportV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[DatasetReadV2]:
+    """
+    Import an existing dataset into a project
+    """
+
+    # Preliminary checks
+    await _get_project_check_owner(
+        project_id=project_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    for image in dataset.images:
+        if not image.zarr_url.startswith(dataset.zarr_dir):
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    f"Cannot import dataset: zarr_url {image.zarr_url} is not "
+                    f"relative to zarr_dir={dataset.zarr_dir}."
+                ),
+            )
+
+    # Create new Dataset
+    db_dataset = DatasetV2(
+        project_id=project_id,
+        **dataset.dict(exclude_none=True),
+    )
+    db.add(db_dataset)
+    await db.commit()
+    await db.refresh(db_dataset)
+    await db.close()
+
+    return db_dataset

fractal_server/app/routes/api/v2/images.py

@@ -0,0 +1,279 @@
+from typing import Any
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from pydantic import BaseModel
+from pydantic import Field
+from sqlalchemy.orm.attributes import flag_modified
+
+from ._aux_functions import _get_dataset_check_owner
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.security import current_active_user
+from fractal_server.app.security import User
+from fractal_server.images import Filters
+from fractal_server.images import SingleImage
+from fractal_server.images import SingleImageUpdate
+from fractal_server.images.tools import find_image_by_zarr_url
+from fractal_server.images.tools import match_filter
+
+router = APIRouter()
+
+
+class ImagePage(BaseModel):
+
+    total_count: int
+    page_size: int
+    current_page: int
+
+    attributes: dict[str, list[Any]]
+    types: list[str]
+
+    images: list[SingleImage]
+
+
+class ImageQuery(BaseModel):
+    zarr_url: Optional[str]
+    filters: Filters = Field(default_factory=Filters)
+
+
+@router.post(
+    "/project/{project_id}/dataset/{dataset_id}/images/",
+    status_code=status.HTTP_201_CREATED,
+)
+async def post_new_image(
+    project_id: int,
+    dataset_id: int,
+    new_image: SingleImage,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+
+    if not new_image.zarr_url.startswith(dataset.zarr_dir):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot create image with zarr_url which is not relative to "
+                f"{dataset.zarr_dir}."
+            ),
+        )
+    elif new_image.zarr_url == dataset.zarr_dir:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "`SingleImage.zarr_url` cannot be equal to `Dataset.zarr_dir`:"
+                f" {dataset.zarr_dir}"
+            ),
+        )
+
+    if new_image.zarr_url in dataset.image_zarr_urls:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Image with zarr_url '{new_image.zarr_url}' "
+                f"already in DatasetV2 {dataset_id}",
+            ),
+        )
+
+    dataset.images.append(new_image.dict())
+    flag_modified(dataset, "images")
+
+    await db.commit()
+
+    return Response(status_code=status.HTTP_201_CREATED)
+
+
+@router.post(
+    "/project/{project_id}/dataset/{dataset_id}/images/query/",
+    response_model=ImagePage,
+    status_code=status.HTTP_200_OK,
+)
+async def query_dataset_images(
+    project_id: int,
+    dataset_id: int,
+    use_dataset_filters: bool = False,  # query param
+    page: int = 1,  # query param
+    page_size: Optional[int] = None,  # query param
+    query: Optional[ImageQuery] = None,  # body
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> ImagePage:
+
+    if page < 1:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Invalid pagination parameter: page={page} < 1",
+        )
+
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+    images = dataset.images
+
+    if use_dataset_filters is True:
+        images = [
+            image
+            for image in images
+            if match_filter(image, Filters(**dataset.filters))
+        ]
+
+    attributes = {}
+    for image in images:
+        for k, v in image["attributes"].items():
+            attributes.setdefault(k, []).append(v)
+        for k, v in attributes.items():
+            attributes[k] = list(set(v))
+
+    types = list(
+        set(type for image in images for type in image["types"].keys())
+    )
+
+    if query is not None:
+
+        if query.zarr_url is not None:
+            image = next(
+                (
+                    image
+                    for image in images
+                    if image["zarr_url"] == query.zarr_url
+                ),
+                None,
+            )
+            if image is None:
+                images = []
+            else:
+                images = [image]
+
+        if query.filters.attributes or query.filters.types:
+            images = [
+                image
+                for image in images
+                if match_filter(
+                    image,
+                    Filters(**query.filters.dict()),
+                )
+            ]
+
+    total_count = len(images)
+
+    if page_size is not None:
+        if page_size <= 0:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    f"Invalid pagination parameter: page_size={page_size} <= 0"
+                ),
+            )
+    else:
+        page_size = total_count
+
+    if total_count == 0:
+        page = 1
+    else:
+        last_page = (total_count // page_size) + (total_count % page_size > 0)
+        if page > last_page:
+            page = last_page
+        offset = (page - 1) * page_size
+        images = images[offset : offset + page_size]  # noqa E203
+
+    return ImagePage(
+        total_count=total_count,
+        current_page=page,
+        page_size=page_size,
+        attributes=attributes,
+        types=types,
+        images=images,
+    )
+
+
+@router.delete(
+    "/project/{project_id}/dataset/{dataset_id}/images/",
+    status_code=status.HTTP_204_NO_CONTENT,
+)
+async def delete_dataset_images(
+    project_id: int,
+    dataset_id: int,
+    zarr_url: str,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+
+    image_to_remove = next(
+        (image for image in dataset.images if image["zarr_url"] == zarr_url),
+        None,
+    )
+    if image_to_remove is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=(
+                f"No image with zarr_url '{zarr_url}' in "
+                f"DatasetV2 {dataset_id}."
+            ),
+        )
+
+    dataset.images.remove(image_to_remove)
+    flag_modified(dataset, "images")
+
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.patch(
+    "/project/{project_id}/dataset/{dataset_id}/images/",
+    response_model=SingleImage,
+    status_code=status.HTTP_200_OK,
+)
+async def patch_dataset_image(
+    project_id: int,
+    dataset_id: int,
+    image_update: SingleImageUpdate,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    db_dataset = output["dataset"]
+
+    ret = find_image_by_zarr_url(
+        images=db_dataset.images, zarr_url=image_update.zarr_url
+    )
+    if ret is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=(
+                f"No image with zarr_url '{image_update.zarr_url}' in "
+                f"DatasetV2 {dataset_id}."
+            ),
+        )
+    index = ret["index"]
+
+    for key, value in image_update.dict(
+        exclude_none=True, exclude={"zarr_url"}
+    ).items():
+        db_dataset.images[index][key] = value
+
+    flag_modified(db_dataset, "images")
+
+    await db.commit()
+    await db.close()
+    return db_dataset.images[index]