fractal-server 2.6.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

fractal_server/app/routes/admin/v2/task_group.py

@@ -0,0 +1,162 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlalchemy.sql.operators import is_
+from sqlalchemy.sql.operators import is_not
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth import current_active_superuser
+from fractal_server.app.routes.auth._aux_auth import (
+    _verify_user_belongs_to_group,
+)
+from fractal_server.app.schemas.v2 import TaskGroupReadV2
+from fractal_server.app.schemas.v2 import TaskGroupUpdateV2
+from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
+from fractal_server.logger import set_logger
+
+router = APIRouter()
+
+logger = set_logger(__name__)
+
+
+@router.get("/{task_group_id}/", response_model=TaskGroupReadV2)
+async def query_task_group(
+    task_group_id: int,
+    user: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskGroupReadV2:
+
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    if task_group is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroup {task_group_id} not found",
+        )
+    return task_group
+
+
+@router.get("/", response_model=list[TaskGroupReadV2])
+async def query_task_group_list(
+    user_id: Optional[int] = None,
+    user_group_id: Optional[int] = None,
+    private: Optional[bool] = None,
+    active: Optional[bool] = None,
+    pkg_name: Optional[str] = None,
+    origin: Optional[TaskGroupV2OriginEnum] = None,
+    user: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskGroupReadV2]:
+
+    stm = select(TaskGroupV2)
+
+    if user_group_id is not None and private is True:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Cannot set `user_group_id` with {private=}",
+        )
+    if user_id is not None:
+        stm = stm.where(TaskGroupV2.user_id == user_id)
+    if user_group_id is not None:
+        stm = stm.where(TaskGroupV2.user_group_id == user_group_id)
+    if private is not None:
+        if private is True:
+            stm = stm.where(is_(TaskGroupV2.user_group_id, None))
+        else:
+            stm = stm.where(is_not(TaskGroupV2.user_group_id, None))
+    if active is not None:
+        if active is True:
+            stm = stm.where(is_(TaskGroupV2.active, True))
+        else:
+            stm = stm.where(is_(TaskGroupV2.active, False))
+    if origin is not None:
+        stm = stm.where(TaskGroupV2.origin == origin)
+    if pkg_name is not None:
+        stm = stm.where(TaskGroupV2.pkg_name.icontains(pkg_name))
+
+    res = await db.execute(stm)
+    task_groups_list = res.scalars().all()
+    return task_groups_list
+
+
+@router.patch("/{task_group_id}/", response_model=TaskGroupReadV2)
+async def patch_task_group(
+    task_group_id: int,
+    task_group_update: TaskGroupUpdateV2,
+    user: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskGroupReadV2]:
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    if task_group is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroupV2 {task_group_id} not found",
+        )
+
+    for key, value in task_group_update.dict(exclude_unset=True).items():
+        if (key == "user_group_id") and (value is not None):
+            await _verify_user_belongs_to_group(
+                user_id=user.id, user_group_id=value, db=db
+            )
+        setattr(task_group, key, value)
+
+    db.add(task_group)
+    await db.commit()
+    await db.refresh(task_group)
+    return task_group
+
+
+@router.delete("/{task_group_id}/", status_code=204)
+async def delete_task_group(
+    task_group_id: int,
+    user: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+):
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    if task_group is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroupV2 {task_group_id} not found",
+        )
+
+    stm = select(WorkflowTaskV2).where(
+        WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
+    )
+    res = await db.execute(stm)
+    workflow_tasks = res.scalars().all()
+    if workflow_tasks != []:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
+        )
+
+    # Cascade operations: set foreign-keys to null for CollectionStateV2 which
+    # are in relationship with the current TaskGroupV2
+    logger.debug("Start of cascade operations on CollectionStateV2.")
+    stm = select(CollectionStateV2).where(
+        CollectionStateV2.taskgroupv2_id == task_group_id
+    )
+    res = await db.execute(stm)
+    collection_states = res.scalars().all()
+    for collection_state in collection_states:
+        logger.debug(
+            f"Setting CollectionStateV2[{collection_state.id}].taskgroupv2_id "
+            "to None."
+        )
+        collection_state.taskgroupv2_id = None
+        db.add(collection_state)
+    logger.debug("End of cascade operations on CollectionStateV2.")
+
+    await db.delete(task_group)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v1/task_collection.py

@@ -26,8 +26,8 @@ from ._aux_functions import _raise_if_v1_is_read_only
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
-from fractal_server.string_tools import slugify_task_name_for_source
-from fractal_server.tasks.utils import get_collection_log
+from fractal_server.string_tools import slugify_task_name_for_source_v1
+from fractal_server.tasks.utils import get_collection_log_v1
 from fractal_server.tasks.v1._TaskCollectPip import _TaskCollectPip
 from fractal_server.tasks.v1.background_operations import (
     background_collect_pip,
@@ -160,7 +160,7 @@ async def collect_tasks_pip(
 
     # Check that tasks are not already in the DB
     for new_task in task_pkg.package_manifest.task_list:
-        new_task_name_slug = slugify_task_name_for_source(new_task.name)
+        new_task_name_slug = slugify_task_name_for_source_v1(new_task.name)
         new_task_source = f"{task_pkg.package_source}:{new_task_name_slug}"
         stm = select(Task).where(Task.source == new_task_source)
         res = await db.execute(stm)
@@ -232,7 +232,7 @@ async def check_collection_status(
     # In some cases (i.e. a successful or ongoing task collection), data.log is
     # not set; if so, we collect the current logs
     if verbose and not data.log:
-        data.log = get_collection_log(data.venv_path)
+        data.log = get_collection_log_v1(data.venv_path)
         state.data = data.sanitised_dict()
     close_logger(logger)
     await db.close()

fractal_server/app/routes/api/v2/__init__.py

@@ -12,7 +12,9 @@ from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
+from .task_group import router as task_group_router_v2
 from .workflow import router as workflow_router_v2
+from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
 from fractal_server.config import get_settings
 from fractal_server.syringe import Inject
@@ -37,6 +39,12 @@ router_api_v2.include_router(
     tags=["V2 Task Collection"],
 )
 router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
+router_api_v2.include_router(
+    task_group_router_v2, prefix="/task-group", tags=["V2 TaskGroup"]
+)
 router_api_v2.include_router(workflow_router_v2, tags=["V2 Workflow"])
+router_api_v2.include_router(
+    workflow_import_router_v2, tags=["V2 Workflow Import"]
+)
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
 router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,8 +21,6 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
-from ...aux.validate_user_settings import verify_user_has_settings
-from fractal_server.app.models import UserOAuth
 from fractal_server.images import Filters
 
 
@@ -320,65 +318,6 @@ async def _get_job_check_owner(
     return dict(job=job, project=project)
 
 
-async def _get_task_check_owner(
-    *,
-    task_id: int,
-    user: UserOAuth,
-    db: AsyncSession,
-) -> TaskV2:
-    """
-    Get a task, after access control.
-
-    This check constitutes a preliminary version of access control:
-    if the current user is not a superuser and differs from the task owner
-    (including when `owner is None`), we raise an 403 HTTP Exception.
-
-    Args:
-        task_id:
-        user:
-        db:
-
-    Returns:
-        The task object.
-
-    Raises:
-        HTTPException(status_code=404_NOT_FOUND):
-            If the task does not exist
-        HTTPException(status_code=403_FORBIDDEN):
-            If the user does not have rights to edit this task.
-    """
-    task = await db.get(TaskV2, task_id)
-    if not task:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"TaskV2 {task_id} not found.",
-        )
-
-    if not user.is_superuser:
-        if task.owner is None:
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail=(
-                    "Only a superuser can modify a TaskV2 with `owner=None`."
-                ),
-            )
-        else:
-            if user.username:
-                owner = user.username
-            else:
-                verify_user_has_settings(user)
-                owner = user.settings.slurm_user
-            if owner != task.owner:
-                raise HTTPException(
-                    status_code=status.HTTP_403_FORBIDDEN,
-                    detail=(
-                        f"Current user ({owner}) cannot modify TaskV2 "
-                        f"{task.id} with different owner ({task.owner})."
-                    ),
-                )
-    return task
-
-
 def _get_submitted_jobs_statement() -> SelectOfScalar:
     """
     Returns:
@@ -393,7 +332,6 @@ async def _workflow_insert_task(
     *,
     workflow_id: int,
     task_id: int,
-    order: Optional[int] = None,
     meta_parallel: Optional[dict[str, Any]] = None,
     meta_non_parallel: Optional[dict[str, Any]] = None,
     args_non_parallel: Optional[dict[str, Any]] = None,
@@ -408,7 +346,6 @@ async def _workflow_insert_task(
         workflow_id:
         task_id:
 
-        order:
         meta_parallel:
         meta_non_parallel:
         args_non_parallel:
@@ -420,9 +357,6 @@ async def _workflow_insert_task(
     if db_workflow is None:
         raise ValueError(f"Workflow {workflow_id} does not exist")
 
-    if order is None:
-        order = len(db_workflow.task_list)
-
     # Get task from db
     db_task = await db.get(TaskV2, task_id)
     if db_task is None:
@@ -458,8 +392,7 @@ async def _workflow_insert_task(
         meta_non_parallel=final_meta_non_parallel,
         **input_filters_kwarg,
     )
-    db_workflow.task_list.insert(order, wf_task)
-    db_workflow.task_list.reorder()  # type: ignore
+    db_workflow.task_list.append(wf_task)
     flag_modified(db_workflow, "task_list")
     await db.commit()
 

fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -0,0 +1,343 @@
+"""
+Auxiliary functions to get task and task-group object from the database or
+perform simple checks
+"""
+from typing import Any
+from typing import Optional
+
+from fastapi import HTTPException
+from fastapi import status
+from sqlmodel import select
+
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models import UserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import TaskV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
+from fractal_server.app.routes.auth._aux_auth import (
+    _verify_user_belongs_to_group,
+)
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
+
+
+async def _get_task_group_or_404(
+    *, task_group_id: int, db: AsyncSession
+) -> TaskGroupV2:
+    """
+    Get an existing task group or raise a 404.
+
+    Arguments:
+        task_group_id: The TaskGroupV2 id
+        db: An asynchronous db session
+    """
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    if task_group is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroupV2 {task_group_id} not found",
+        )
+    return task_group
+
+
+async def _get_task_group_read_access(
+    *,
+    task_group_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Get a task group or raise a 403 if user has no read access.
+
+    Arguments:
+        task_group_id: ID of the required task group.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+    """
+    task_group = await _get_task_group_or_404(
+        task_group_id=task_group_id, db=db
+    )
+
+    # Prepare exception to be used below
+    forbidden_exception = HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail=(
+            "Current user has no read access to TaskGroupV2 "
+            f"{task_group_id}.",
+        ),
+    )
+
+    if task_group.user_id == user_id:
+        return task_group
+    elif task_group.user_group_id is None:
+        raise forbidden_exception
+    else:
+        stm = (
+            select(LinkUserGroup)
+            .where(LinkUserGroup.group_id == task_group.user_group_id)
+            .where(LinkUserGroup.user_id == user_id)
+        )
+        res = await db.execute(stm)
+        link = res.scalar_one_or_none()
+        if link is None:
+            raise forbidden_exception
+        else:
+            return task_group
+
+
+async def _get_task_group_full_access(
+    *,
+    task_group_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Get a task group or raise a 403 if user has no full access.
+
+    Arguments:
+        task_group_id: ID of the required task group.
+        user_id: ID of the current user.
+        db: An asynchronous db session
+    """
+    task_group = await _get_task_group_or_404(
+        task_group_id=task_group_id, db=db
+    )
+
+    if task_group.user_id == user_id:
+        return task_group
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=(
+                "Current user has no full access to "
+                f"TaskGroupV2 {task_group_id}.",
+            ),
+        )
+
+
+async def _get_task_or_404(*, task_id: int, db: AsyncSession) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        db: An asynchronous db session
+    """
+    task = await db.get(TaskV2, task_id)
+    if task is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskV2 {task_id} not found",
+        )
+    return task
+
+
+async def _get_task_full_access(
+    *,
+    task_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+    """
+    task = await _get_task_or_404(task_id=task_id, db=db)
+    await _get_task_group_full_access(
+        task_group_id=task.taskgroupv2_id, user_id=user_id, db=db
+    )
+    return task
+
+
+async def _get_task_read_access(
+    *,
+    task_id: int,
+    user_id: int,
+    db: AsyncSession,
+    require_active: bool = False,
+) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+        require_active: If set, fail when the task group is not `active`
+    """
+    task = await _get_task_or_404(task_id=task_id, db=db)
+    task_group = await _get_task_group_read_access(
+        task_group_id=task.taskgroupv2_id, user_id=user_id, db=db
+    )
+    if require_active:
+        if not task_group.active:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Error: task {task_id} ({task.name}) is not active.",
+            )
+    return task
+
+
+async def _get_valid_user_group_id(
+    *,
+    user_group_id: Optional[int] = None,
+    private: bool,
+    user_id: int,
+    db: AsyncSession,
+) -> Optional[int]:
+    """
+    Validate query parameters for endpoints that create some task(s).
+
+    Arguments:
+        user_group_id:
+        private:
+        user_id: ID of the current user
+        db: An asynchronous db session.
+    """
+    if (user_group_id is not None) and (private is True):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Cannot set both {user_group_id=} and {private=}",
+        )
+    elif private is True:
+        user_group_id = None
+    elif user_group_id is None:
+        user_group_id = await _get_default_usergroup_id(db=db)
+    else:
+        await _verify_user_belongs_to_group(
+            user_id=user_id, user_group_id=user_group_id, db=db
+        )
+    return user_group_id
+
+
+async def _get_collection_status_message(
+    task_group: TaskGroupV2, db: AsyncSession
+) -> str:
+    res = await db.execute(
+        select(CollectionStateV2).where(
+            CollectionStateV2.taskgroupv2_id == task_group.id
+        )
+    )
+    states = res.scalars().all()
+    if len(states) > 1:
+        msg = (
+            "Expected one CollectionStateV2 associated to TaskGroup "
+            f"{task_group.id}, found {len(states)} "
+            f"(IDs: {[state.id for state in states]}).\n"
+            "Warning: this should have not happened, please contact an admin."
+        )
+    elif len(states) == 1:
+        msg = (
+            f"\nThere exists a task-collection state (ID={states[0].id}) for "
+            f"such task group (ID={task_group.id}), with status "
+            f"'{states[0].data.get('status')}'."
+        )
+    else:
+        msg = ""
+    return msg
+
+
+async def _verify_non_duplication_user_constraint(
+    db: AsyncSession,
+    user_id: int,
+    pkg_name: str,
+    version: Optional[str],
+):
+    stm = (
+        select(TaskGroupV2)
+        .where(TaskGroupV2.user_id == user_id)
+        .where(TaskGroupV2.pkg_name == pkg_name)
+        .where(TaskGroupV2.version == version)
+    )
+    res = await db.execute(stm)
+    duplicate = res.scalars().all()
+    if duplicate:
+        user = await db.get(UserOAuth, user_id)
+        if len(duplicate) > 1:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Invalid state:\n"
+                    f"User '{user.email}' already owns {len(duplicate)} task "
+                    f"groups with name='{pkg_name}' and {version=} "
+                    f"(IDs: {[group.id for group in duplicate]}).\n"
+                    "This should have not happened: please contact an admin."
+                ),
+            )
+        state_msg = await _get_collection_status_message(duplicate[0], db)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"User '{user.email}' already owns a task group "
+                f"with name='{pkg_name}' and {version=}.{state_msg}"
+            ),
+        )
+
+
+async def _verify_non_duplication_group_constraint(
+    db: AsyncSession,
+    user_group_id: Optional[int],
+    pkg_name: str,
+    version: Optional[str],
+):
+    if user_group_id is None:
+        return
+
+    stm = (
+        select(TaskGroupV2)
+        .where(TaskGroupV2.user_group_id == user_group_id)
+        .where(TaskGroupV2.pkg_name == pkg_name)
+        .where(TaskGroupV2.version == version)
+    )
+    res = await db.execute(stm)
+    duplicate = res.scalars().all()
+    if duplicate:
+        user_group = await db.get(UserGroup, user_group_id)
+        if len(duplicate) > 1:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Invalid state:\n"
+                    f"UserGroup '{user_group.name}' already owns "
+                    f"{len(duplicate)} task groups with name='{pkg_name}' and "
+                    f"{version=} (IDs: {[group.id for group in duplicate]}).\n"
+                    "This should have not happened: please contact an admin."
+                ),
+            )
+        state_msg = await _get_collection_status_message(duplicate[0], db)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"UserGroup {user_group.name} already owns a task group "
+                f"with {pkg_name=} and {version=}.{state_msg}"
+            ),
+        )
+
+
+async def _add_warnings_to_workflow_tasks(
+    wftask_list: list[WorkflowTaskV2], user_id: int, db: AsyncSession
+) -> list[dict[str, Any]]:
+    wftask_list_with_warnings = []
+    for wftask in wftask_list:
+        wftask_data = dict(wftask.model_dump(), task=wftask.task)
+        try:
+            task_group = await _get_task_group_read_access(
+                task_group_id=wftask.task.taskgroupv2_id,
+                user_id=user_id,
+                db=db,
+            )
+            if not task_group.active:
+                wftask_data["warning"] = "Task is not active."
+        except HTTPException:
+            wftask_data["warning"] = "Current user has no access to this task."
+        wftask_list_with_warnings.append(wftask_data)
+    return wftask_list_with_warnings