fractal-server 2.6.3__py3-none-any.whl → 2.7.0a0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.6.3"
+__VERSION__ = "2.7.0a0"

fractal_server/app/models/v2/__init__.py

@@ -6,6 +6,7 @@ from .collection_state import CollectionStateV2
 from .dataset import DatasetV2
 from .job import JobV2
 from .project import ProjectV2
+from .task import TaskGroupV2
 from .task import TaskV2
 from .workflow import WorkflowV2
 from .workflowtask import WorkflowTaskV2
@@ -16,6 +17,7 @@ __all__ = [
     "JobV2",
     "ProjectV2",
     "CollectionStateV2",
+    "TaskGroupV2",
     "TaskV2",
     "WorkflowTaskV2",
     "WorkflowV2",

fractal_server/app/models/v2/task.py

@@ -1,12 +1,17 @@
+from datetime import datetime
 from typing import Any
 from typing import Optional
 
 from pydantic import HttpUrl
 from sqlalchemy import Column
+from sqlalchemy.types import DateTime
 from sqlalchemy.types import JSON
 from sqlmodel import Field
+from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
+from fractal_server.utils import get_timestamp
+
 
 class TaskV2(SQLModel, table=True):
 
@@ -39,3 +44,25 @@ class TaskV2(SQLModel, table=True):
 
     input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
     output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
+
+    taskgroupv2_id: Optional[int] = Field(foreign_key="taskgroupv2.id")
+
+
+class TaskGroupV2(SQLModel, table=True):
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    user_id: int = Field(foreign_key="user_oauth.id")
+    user_group_id: Optional[int] = Field(foreign_key="usergroup.id")
+
+    active: bool = True
+    task_list: list[TaskV2] = Relationship(
+        sa_relationship_kwargs=dict(
+            lazy="selectin", cascade="all, delete-orphan"
+        ),
+    )
+
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )

fractal_server/app/routes/api/v2/__init__.py

@@ -12,6 +12,7 @@ from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
+from .task_group import router as task_group_router_v2
 from .workflow import router as workflow_router_v2
 from .workflowtask import router as workflowtask_router_v2
 from fractal_server.config import get_settings
@@ -37,6 +38,9 @@ router_api_v2.include_router(
     tags=["V2 Task Collection"],
 )
 router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
+router_api_v2.include_router(
+    task_group_router_v2, prefix="/task-group", tags=["V2 TaskGroup"]
+)
 router_api_v2.include_router(workflow_router_v2, tags=["V2 Workflow"])
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
 router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,8 +21,6 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
-from ...aux.validate_user_settings import verify_user_has_settings
-from fractal_server.app.models import UserOAuth
 from fractal_server.images import Filters
 
 
@@ -320,65 +318,6 @@ async def _get_job_check_owner(
     return dict(job=job, project=project)
 
 
-async def _get_task_check_owner(
-    *,
-    task_id: int,
-    user: UserOAuth,
-    db: AsyncSession,
-) -> TaskV2:
-    """
-    Get a task, after access control.
-
-    This check constitutes a preliminary version of access control:
-    if the current user is not a superuser and differs from the task owner
-    (including when `owner is None`), we raise an 403 HTTP Exception.
-
-    Args:
-        task_id:
-        user:
-        db:
-
-    Returns:
-        The task object.
-
-    Raises:
-        HTTPException(status_code=404_NOT_FOUND):
-            If the task does not exist
-        HTTPException(status_code=403_FORBIDDEN):
-            If the user does not have rights to edit this task.
-    """
-    task = await db.get(TaskV2, task_id)
-    if not task:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"TaskV2 {task_id} not found.",
-        )
-
-    if not user.is_superuser:
-        if task.owner is None:
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail=(
-                    "Only a superuser can modify a TaskV2 with `owner=None`."
-                ),
-            )
-        else:
-            if user.username:
-                owner = user.username
-            else:
-                verify_user_has_settings(user)
-                owner = user.settings.slurm_user
-            if owner != task.owner:
-                raise HTTPException(
-                    status_code=status.HTTP_403_FORBIDDEN,
-                    detail=(
-                        f"Current user ({owner}) cannot modify TaskV2 "
-                        f"{task.id} with different owner ({task.owner})."
-                    ),
-                )
-    return task
-
-
 def _get_submitted_jobs_statement() -> SelectOfScalar:
     """
     Returns:

fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -0,0 +1,209 @@
+"""
+Auxiliary functions to get task and task-group object from the database or
+perform simple checks
+"""
+from typing import Optional
+
+from fastapi import HTTPException
+from fastapi import status
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....models import LinkUserGroup
+from ....models.v2 import TaskGroupV2
+from ....models.v2 import TaskV2
+from ...auth._aux_auth import _get_default_user_group_id
+from ...auth._aux_auth import _verify_user_belongs_to_group
+
+
+async def _get_task_group_or_404(
+    *, task_group_id: int, db: AsyncSession
+) -> TaskGroupV2:
+    """
+    Get an existing task group or raise a 404.
+
+    Arguments:
+        task_group_id: The TaskGroupV2 id
+        db: An asynchronous db session
+    """
+    task_group = await db.get(TaskGroupV2, task_group_id)
+    if task_group is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroupV2 {task_group_id} not found",
+        )
+    return task_group
+
+
+async def _get_task_group_read_access(
+    *,
+    task_group_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Get a task group or raise a 403 if user has no read access.
+
+    Arguments:
+        task_group_id: ID of the required task group.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+    """
+    task_group = await _get_task_group_or_404(
+        task_group_id=task_group_id, db=db
+    )
+
+    # Prepare exception to be used below
+    forbidden_exception = HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail=(
+            "Current user has no read access to TaskGroupV2 "
+            f"{task_group_id}.",
+        ),
+    )
+
+    if task_group.user_id == user_id:
+        return task_group
+    elif task_group.user_group_id is None:
+        raise forbidden_exception
+    else:
+        stm = (
+            select(LinkUserGroup)
+            .where(LinkUserGroup.group_id == task_group.user_group_id)
+            .where(LinkUserGroup.user_id == user_id)
+        )
+        res = await db.execute(stm)
+        link = res.scalar_one_or_none()
+        if link is None:
+            raise forbidden_exception
+        else:
+            return task_group
+
+
+async def _get_task_group_full_access(
+    *,
+    task_group_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskGroupV2:
+    """
+    Get a task group or raise a 403 if user has no full access.
+
+    Arguments:
+        task_group_id: ID of the required task group.
+        user_id: ID of the current user.
+        db: An asynchronous db session
+    """
+    task_group = await _get_task_group_or_404(
+        task_group_id=task_group_id, db=db
+    )
+
+    if task_group.user_id == user_id:
+        return task_group
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=(
+                "Current user has no full access to "
+                f"TaskGroupV2 {task_group_id}.",
+            ),
+        )
+
+
+async def _get_task_or_404(*, task_id: int, db: AsyncSession) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        db: An asynchronous db session
+    """
+    task = await db.get(TaskV2, task_id)
+    if task is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskV2 {task_id} not found",
+        )
+    return task
+
+
+async def _get_task_full_access(
+    *,
+    task_id: int,
+    user_id: int,
+    db: AsyncSession,
+) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+    """
+    task = await _get_task_or_404(task_id=task_id, db=db)
+    await _get_task_group_full_access(
+        task_group_id=task.taskgroupv2_id, user_id=user_id, db=db
+    )
+    return task
+
+
+async def _get_task_read_access(
+    *,
+    task_id: int,
+    user_id: int,
+    db: AsyncSession,
+    require_active: bool = False,
+) -> TaskV2:
+    """
+    Get an existing task or raise a 404.
+
+    Arguments:
+        task_id: ID of the required task.
+        user_id: ID of the current user.
+        db: An asynchronous db session.
+        require_active: If set, fail when the task group is not `active`
+    """
+    task = await _get_task_or_404(task_id=task_id, db=db)
+    task_group = await _get_task_group_read_access(
+        task_group_id=task.taskgroupv2_id, user_id=user_id, db=db
+    )
+    if require_active:
+        if not task_group.active:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail="Cannot insert non-active tasks into a workflow.",
+            )
+    return task
+
+
+async def _get_valid_user_group_id(
+    *,
+    user_group_id: Optional[int] = None,
+    private: bool,
+    user_id: int,
+    db: AsyncSession,
+) -> Optional[int]:
+    """
+    Validate query parameters for endpoints that create some task(s).
+
+    Arguments:
+        user_group_id:
+        private:
+        user_id: ID of the current user
+        db: An asynchronous db session.
+    """
+    if (user_group_id is not None) and (private is True):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Cannot set both {user_group_id=} and {private=}",
+        )
+    elif private is True:
+        user_group_id = None
+    elif user_group_id is None:
+        user_group_id = await _get_default_user_group_id(db=db)
+    else:
+        await _verify_user_belongs_to_group(
+            user_id=user_id, user_group_id=user_group_id, db=db
+        )
+    return user_group_id

fractal_server/app/routes/api/v2/submit.py

@@ -32,6 +32,9 @@ from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.api.v2._aux_functions_tasks import (
+    _get_task_read_access,
+)
 from fractal_server.app.routes.auth import current_active_verified_user
 
 
@@ -83,15 +86,14 @@ async def apply_workflow(
     workflow = await _get_workflow_check_owner(
         project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
     )
-
-    if not workflow.task_list:
+    num_tasks = len(workflow.task_list)
+    if num_tasks == 0:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Workflow {workflow_id} has empty task list",
         )
 
     # Set values of first_task_index and last_task_index
-    num_tasks = len(workflow.task_list)
     try:
         first_task_index, last_task_index = set_start_and_last_task_index(
             num_tasks,
@@ -110,6 +112,17 @@ async def apply_workflow(
             ),
         )
 
+    # Check that tasks have read-access and are `active`
+    for wftask in workflow.task_list[
+        first_task_index : last_task_index + 1  # noqa: E203
+    ]:
+        await _get_task_read_access(
+            user_id=user.id,
+            task_id=wftask.task_id,
+            require_active=True,
+            db=db,
+        )
+
     # Validate user settings
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     user_settings = await validate_user_settings(

fractal_server/app/routes/api/v2/task.py

@@ -6,23 +6,26 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import or_
 from sqlmodel import select
 
-from .....logger import set_logger
-from ....db import AsyncSession
-from ....db import get_async_db
-from ....models.v1 import Task as TaskV1
-from ....models.v2 import TaskV2
-from ....models.v2 import WorkflowTaskV2
-from ....models.v2 import WorkflowV2
-from ....schemas.v2 import TaskCreateV2
-from ....schemas.v2 import TaskReadV2
-from ....schemas.v2 import TaskUpdateV2
 from ...aux.validate_user_settings import verify_user_has_settings
-from ._aux_functions import _get_task_check_owner
+from ._aux_functions_tasks import _get_task_full_access
+from ._aux_functions_tasks import _get_task_read_access
+from ._aux_functions_tasks import _get_valid_user_group_id
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v1 import Task as TaskV1
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.schemas.v2 import TaskCreateV2
+from fractal_server.app.schemas.v2 import TaskReadV2
+from fractal_server.app.schemas.v2 import TaskUpdateV2
+from fractal_server.logger import set_logger
 
 router = APIRouter()
 
@@ -39,7 +42,21 @@ async def get_list_task(
     """
     Get list of available tasks
     """
-    stm = select(TaskV2)
+    stm = (
+        select(TaskV2)
+        .join(TaskGroupV2)
+        .where(TaskGroupV2.id == TaskV2.taskgroupv2_id)
+        .where(
+            or_(
+                TaskGroupV2.user_id == user.id,
+                TaskGroupV2.user_group_id.in_(
+                    select(LinkUserGroup.group_id).where(
+                        LinkUserGroup.user_id == user.id
+                    )
+                ),
+            )
+        )
+    )
     res = await db.execute(stm)
     task_list = res.scalars().all()
     await db.close()
@@ -62,12 +79,7 @@ async def get_task(
     """
     Get info on a specific task
     """
-    task = await db.get(TaskV2, task_id)
-    await db.close()
-    if not task:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND, detail="TaskV2 not found"
-        )
+    task = await _get_task_read_access(task_id=task_id, user_id=user.id, db=db)
     return task
 
 
@@ -83,7 +95,9 @@ async def patch_task(
     """
 
     # Retrieve task from database
-    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
+    db_task = await _get_task_full_access(
+        task_id=task_id, user_id=user.id, db=db
+    )
     update = task_update.dict(exclude_unset=True)
 
     # Forbid changes that set a previously unset command
@@ -112,6 +126,8 @@ async def patch_task(
 )
 async def create_task(
     task: TaskCreateV2,
+    user_group_id: Optional[int] = None,
+    private: bool = False,
     user: UserOAuth = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[TaskReadV2]:
@@ -119,6 +135,14 @@ async def create_task(
     Create a new task
     """
 
+    # Validate query parameters related to user-group ownership
+    user_group_id = await _get_valid_user_group_id(
+        user_group_id=user_group_id,
+        private=private,
+        user_id=user.id,
+        db=db,
+    )
+
     if task.command_non_parallel is None:
         task_type = "parallel"
     elif task.command_parallel is None:
@@ -148,7 +172,7 @@ async def create_task(
             ),
         )
 
-    # Set task.owner attribute
+    # Set task.owner attribute - FIXME: remove this block
     if user.username:
         owner = user.username
     else:
@@ -186,7 +210,14 @@ async def create_task(
         )
     # Add task
     db_task = TaskV2(**task.dict(), owner=owner, type=task_type)
-    db.add(db_task)
+
+    db_task_group = TaskGroupV2(
+        user_id=user.id,
+        user_group_id=user_group_id,
+        active=True,
+        task_list=[db_task],
+    )
+    db.add(db_task_group)
     await db.commit()
     await db.refresh(db_task)
     await db.close()
@@ -202,54 +233,10 @@ async def delete_task(
     """
     Delete a task
     """
-
-    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
-
-    # Check that the TaskV2 is not in relationship with some WorkflowTaskV2
-    stm = select(WorkflowTaskV2).filter(WorkflowTaskV2.task_id == task_id)
-    res = await db.execute(stm)
-    workflow_tasks = res.scalars().all()
-
-    if workflow_tasks:
-        # Find IDs of all affected workflows
-        workflow_ids = set(wftask.workflow_id for wftask in workflow_tasks)
-        # Fetch all affected workflows from DB
-        stm = select(WorkflowV2).where(WorkflowV2.id.in_(workflow_ids))
-        res = await db.execute(stm)
-        workflows = res.scalars().all()
-
-        # Find which workflows are associated to the current user
-        workflows_current_user = [
-            wf for wf in workflows if user in wf.project.user_list
-        ]
-        if workflows_current_user:
-            current_user_msg = (
-                "For the current-user workflows (listed below),"
-                " you can update the task or remove the workflows.\n"
-            )
-            current_user_msg += "\n".join(
-                [
-                    f"* '{wf.name}' (id={wf.id})"
-                    for wf in workflows_current_user
-                ]
-            )
-        else:
-            current_user_msg = ""
-
-        # Count workflows of current users or other users
-        num_workflows_current_user = len(workflows_current_user)
-        num_workflows_other_users = len(workflows) - num_workflows_current_user
-
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=(
-                f"Cannot remove Task with id={task_id}: it is currently in "
-                f"use in {num_workflows_current_user} current-user workflows "
-                f"and in {num_workflows_other_users} other-users workflows.\n"
-                f"{current_user_msg}"
-            ),
-        )
-
-    await db.delete(db_task)
-    await db.commit()
-    return Response(status_code=status.HTTP_204_NO_CONTENT)
+    raise HTTPException(
+        status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
+        detail=(
+            "Cannot delete single tasks, "
+            "please operate directly on task groups."
+        ),
+    )

fractal_server/app/routes/api/v2/task_collection.py

@@ -2,6 +2,7 @@ import json
 from pathlib import Path
 from shutil import copy as shell_copy
 from tempfile import TemporaryDirectory
+from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import BackgroundTasks
@@ -26,6 +27,7 @@ from ....schemas.v2 import CollectionStatusV2
 from ....schemas.v2 import TaskCollectPipV2
 from ....schemas.v2 import TaskReadV2
 from ...aux.validate_user_settings import validate_user_settings
+from ._aux_functions_tasks import _get_valid_user_group_id
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
@@ -69,6 +71,8 @@ async def collect_tasks_pip(
     background_tasks: BackgroundTasks,
     response: Response,
     request: Request,
+    private: bool = False,
+    user_group_id: Optional[int] = None,
     user: UserOAuth = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> CollectionStateReadV2:
@@ -112,6 +116,14 @@ async def collect_tasks_pip(
         user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
     )
 
+    # Validate query parameters related to user-group ownership
+    user_group_id = await _get_valid_user_group_id(
+        user_group_id=user_group_id,
+        private=private,
+        user_id=user.id,
+        db=db,
+    )
+
     # END of SSH/non-SSH common part
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
@@ -140,10 +152,12 @@ async def collect_tasks_pip(
 
         background_tasks.add_task(
             background_collect_pip_ssh,
-            state.id,
-            task_pkg,
-            fractal_ssh,
-            user_settings.ssh_tasks_dir,
+            state_id=state.id,
+            task_pkg=task_pkg,
+            fractal_ssh=fractal_ssh,
+            tasks_base_dir=user_settings.ssh_tasks_dir,
+            user_id=user.id,
+            user_group_id=user_group_id,
         )
 
         response.status_code = status.HTTP_201_CREATED
@@ -284,6 +298,8 @@ async def collect_tasks_pip(
         state_id=state.id,
         venv_path=venv_path,
         task_pkg=task_pkg,
+        user_id=user.id,
+        user_group_id=user_group_id,
     )
     logger.debug(
         "Task-collection endpoint: start background collection "