fractal-server 2.6.4__py3-none-any.whl → 2.7.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -1,31 +1,40 @@
 import shlex
 import subprocess  # nosec
 from pathlib import Path
+from typing import Optional
 
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import status
+from sqlalchemy.ext.asyncio import AsyncSession
 from sqlmodel import select
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ....db import DBSyncSession
-from ....db import get_sync_db
-from ....models.v1 import Task as TaskV1
-from ....models.v2 import TaskV2
-from ....schemas.v2 import TaskCollectCustomV2
-from ....schemas.v2 import TaskCreateV2
-from ....schemas.v2 import TaskReadV2
-from ...aux.validate_user_settings import verify_user_has_settings
+from ._aux_functions_tasks import _get_valid_user_group_id
+from fractal_server.app.db import DBSyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v1 import Task as TaskV1
+from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.routes.aux.validate_user_settings import (
+    verify_user_has_settings,
+)
+from fractal_server.app.schemas.v2 import TaskCollectCustomV2
+from fractal_server.app.schemas.v2 import TaskCreateV2
+from fractal_server.app.schemas.v2 import TaskGroupCreateV2
+from fractal_server.app.schemas.v2 import TaskReadV2
+from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
-from fractal_server.tasks.v2.background_operations import _insert_tasks
+from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.background_operations import (
     _prepare_tasks_metadata,
 )
+from fractal_server.tasks.v2.database_operations import (
+    create_db_task_group_and_tasks,
+)
 
 router = APIRouter()
 
@@ -37,12 +46,25 @@ logger = set_logger(__name__)
 )
 async def collect_task_custom(
     task_collect: TaskCollectCustomV2,
+    private: bool = False,
+    user_group_id: Optional[int] = None,
     user: UserOAuth = Depends(current_active_verified_user),
-    db: DBSyncSession = Depends(get_sync_db),
+    db: AsyncSession = Depends(get_async_db),  # FIXME: using both sync/async
+    db_sync: DBSyncSession = Depends(
+        get_sync_db
+    ),  # FIXME: using both sync/async
 ) -> list[TaskReadV2]:
 
     settings = Inject(get_settings)
 
+    # Validate query parameters related to user-group ownership
+    user_group_id = await _get_valid_user_group_id(
+        user_group_id=user_group_id,
+        private=private,
+        user_id=user.id,
+        db=db,
+    )
+
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         if task_collect.package_root is None:
             raise HTTPException(
@@ -140,7 +162,7 @@ async def collect_task_custom(
     # already guaranteed by a constraint in the table definition).
     sources = [task.source for task in task_list]
     stm = select(TaskV2).where(TaskV2.source.in_(sources))
-    res = db.execute(stm)
+    res = db_sync.execute(stm)
     overlapping_sources_v2 = res.scalars().all()
     if overlapping_sources_v2:
         overlapping_tasks_v2_source_and_id = [
@@ -152,7 +174,7 @@ async def collect_task_custom(
             detail="\n".join(overlapping_tasks_v2_source_and_id),
         )
     stm = select(TaskV1).where(TaskV1.source.in_(sources))
-    res = db.execute(stm)
+    res = db_sync.execute(stm)
     overlapping_sources_v1 = res.scalars().all()
     if overlapping_sources_v1:
         overlapping_tasks_v1_source_and_id = [
@@ -164,8 +186,12 @@ async def collect_task_custom(
             detail="\n".join(overlapping_tasks_v1_source_and_id),
         )
 
-    task_list_db: list[TaskV2] = _insert_tasks(
-        task_list=task_list, owner=owner, db=db
+    task_group = create_db_task_group_and_tasks(
+        task_list=task_list,
+        task_group_obj=TaskGroupCreateV2(),
+        user_id=user.id,
+        user_group_id=user_group_id,
+        db=db_sync,
     )
 
     logger.debug(
@@ -173,4 +199,4 @@ async def collect_task_custom(
         f"for package with {source=}"
     )
 
-    return task_list_db
+    return task_group.task_list

fractal_server/app/routes/api/v2/task_group.py

@@ -0,0 +1,130 @@
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import or_
+from sqlmodel import select
+
+from ._aux_functions_tasks import _get_task_group_full_access
+from ._aux_functions_tasks import _get_task_group_read_access
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.auth._aux_auth import (
+    _verify_user_belongs_to_group,
+)
+from fractal_server.app.schemas.v2 import TaskGroupReadV2
+from fractal_server.app.schemas.v2 import TaskGroupUpdateV2
+from fractal_server.logger import set_logger
+
+router = APIRouter()
+
+logger = set_logger(__name__)
+
+
+@router.get("/", response_model=list[TaskGroupReadV2])
+async def get_task_group_list(
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskGroupReadV2]:
+    """
+    Get all accessible TaskGroups
+    """
+    stm = select(TaskGroupV2).where(
+        or_(
+            TaskGroupV2.user_id == user.id,
+            TaskGroupV2.user_group_id.in_(
+                select(LinkUserGroup.group_id).where(
+                    LinkUserGroup.user_id == user.id
+                )
+            ),
+        )
+    )
+    res = await db.execute(stm)
+    task_groups = res.scalars().all()
+
+    return task_groups
+
+
+@router.get("/{task_group_id}/", response_model=TaskGroupReadV2)
+async def get_task_group(
+    task_group_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskGroupReadV2:
+    """
+    Get single TaskGroup
+    """
+    task_group = await _get_task_group_read_access(
+        task_group_id=task_group_id,
+        user_id=user.id,
+        db=db,
+    )
+    return task_group
+
+
+@router.delete("/{task_group_id}/", status_code=204)
+async def delete_task_group(
+    task_group_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    """
+    Delete single TaskGroup
+    """
+
+    task_group = await _get_task_group_full_access(
+        task_group_id=task_group_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    stm = select(WorkflowTaskV2).where(
+        WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
+    )
+    res = await db.execute(stm)
+    workflow_tasks = res.scalars().all()
+    if workflow_tasks != []:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
+        )
+
+    await db.delete(task_group)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.patch("/{task_group_id}/", response_model=TaskGroupReadV2)
+async def patch_task_group(
+    task_group_id: int,
+    task_group_update: TaskGroupUpdateV2,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskGroupReadV2:
+    """
+    Patch single TaskGroup
+    """
+    task_group = await _get_task_group_full_access(
+        task_group_id=task_group_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    for key, value in task_group_update.dict(exclude_unset=True).items():
+        if (key == "user_group_id") and (value is not None):
+            await _verify_user_belongs_to_group(
+                user_id=user.id, user_group_id=value, db=db
+            )
+        setattr(task_group, key, value)
+
+    db.add(task_group)
+    await db.commit()
+    await db.refresh(task_group)
+    return task_group

fractal_server/app/routes/api/v2/workflow.py

@@ -19,6 +19,7 @@ from ....schemas.v2 import WorkflowCreateV2
 from ....schemas.v2 import WorkflowExportV2
 from ....schemas.v2 import WorkflowImportV2
 from ....schemas.v2 import WorkflowReadV2
+from ....schemas.v2 import WorkflowReadV2WithWarnings
 from ....schemas.v2 import WorkflowTaskCreateV2
 from ....schemas.v2 import WorkflowUpdateV2
 from ._aux_functions import _check_workflow_exists
@@ -26,6 +27,7 @@ from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _workflow_insert_task
+from ._aux_functions_tasks import _get_task_group_read_access
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 
@@ -89,14 +91,14 @@ async def create_workflow(
 
 @router.get(
     "/project/{project_id}/workflow/{workflow_id}/",
-    response_model=WorkflowReadV2,
+    response_model=WorkflowReadV2WithWarnings,
 )
 async def read_workflow(
     project_id: int,
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[WorkflowReadV2]:
+) -> Optional[WorkflowReadV2WithWarnings]:
     """
     Get info on an existing workflow
     """
@@ -108,7 +110,26 @@ async def read_workflow(
         db=db,
     )
 
-    return workflow
+    workflow_data = dict(
+        **workflow.model_dump(),
+        project=workflow.project,
+    )
+    task_list_with_warnings = []
+    for wftask in workflow.task_list:
+        wftask_data = dict(wftask.model_dump(), task=wftask.task)
+        try:
+            task_group = await _get_task_group_read_access(
+                task_group_id=wftask.task.taskgroupv2_id,
+                user_id=user.id,
+                db=db,
+            )
+            if not task_group.active:
+                wftask_data["warning"] = "Task is not active."
+        except HTTPException:
+            wftask_data["warning"] = "Current user has no access to this task."
+        task_list_with_warnings.append(wftask_data)
+    workflow_data["task_list"] = task_list_with_warnings
+    return workflow_data
 
 
 @router.patch(

fractal_server/app/routes/api/v2/workflowtask.py

@@ -9,13 +9,13 @@ from fastapi import status
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models.v2 import TaskV2
 from ....schemas.v2 import WorkflowTaskCreateV2
 from ....schemas.v2 import WorkflowTaskReadV2
 from ....schemas.v2 import WorkflowTaskUpdateV2
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import _get_workflow_task_check_owner
 from ._aux_functions import _workflow_insert_task
+from ._aux_functions_tasks import _get_task_read_access
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 
@@ -43,12 +43,9 @@ async def create_workflowtask(
         project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
     )
 
-    task = await db.get(TaskV2, task_id)
-    if not task:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"TaskV2 {task_id} not found.",
-        )
+    task = await _get_task_read_access(
+        task_id=task_id, user_id=user.id, db=db, require_active=True
+    )
 
     if task.type == "parallel":
         if (

fractal_server/app/routes/auth/_aux_auth.py

@@ -8,6 +8,7 @@ from fractal_server.app.models.security import UserGroup
 from fractal_server.app.models.security import UserOAuth
 from fractal_server.app.schemas.user import UserRead
 from fractal_server.app.schemas.user_group import UserGroupRead
+from fractal_server.app.security import FRACTAL_DEFAULT_GROUP_NAME
 
 
 async def _get_single_user_with_group_names(
@@ -111,3 +112,44 @@ async def _user_or_404(user_id: int, db: AsyncSession) -> UserOAuth:
             status_code=status.HTTP_404_NOT_FOUND, detail="User not found."
         )
     return user
+
+
+async def _get_default_user_group_id(db: AsyncSession) -> int:
+    stm = select(UserGroup.id).where(
+        UserGroup.name == FRACTAL_DEFAULT_GROUP_NAME
+    )
+    res = await db.execute(stm)
+    user_group_id = res.scalars().one_or_none()
+    if user_group_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"User group '{FRACTAL_DEFAULT_GROUP_NAME}' not found.",
+        )
+    return user_group_id
+
+
+async def _verify_user_belongs_to_group(
+    *, user_id: int, user_group_id: int, db: AsyncSession
+):
+    stm = (
+        select(LinkUserGroup)
+        .where(LinkUserGroup.user_id == user_id)
+        .where(LinkUserGroup.group_id == user_group_id)
+    )
+    res = await db.execute(stm)
+    link = res.scalars().one_or_none()
+    if link is None:
+        group = await db.get(UserGroup, user_group_id)
+        if group is None:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"UserGroup {user_group_id} not found",
+            )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=(
+                    f"User {user_id} does not belong "
+                    f"to UserGroup {user_group_id}"
+                ),
+            )

fractal_server/app/schemas/v2/__init__.py

@@ -26,14 +26,19 @@ from .task_collection import CollectionStateReadV2  # noqa F401
 from .task_collection import CollectionStatusV2  # noqa F401
 from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401
+from .task_group import TaskGroupCreateV2  # noqa F401
+from .task_group import TaskGroupReadV2  # noqa F401
+from .task_group import TaskGroupUpdateV2  # noqa F401
 from .workflow import WorkflowCreateV2  # noqa F401
 from .workflow import WorkflowExportV2  # noqa F401
 from .workflow import WorkflowImportV2  # noqa F401
 from .workflow import WorkflowReadV2  # noqa F401
+from .workflow import WorkflowReadV2WithWarnings  # noqa F401
 from .workflow import WorkflowUpdateV2  # noqa F401
 from .workflowtask import WorkflowTaskCreateV2  # noqa F401
 from .workflowtask import WorkflowTaskExportV2  # noqa F401
 from .workflowtask import WorkflowTaskImportV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2  # noqa F401
+from .workflowtask import WorkflowTaskReadV2WithWarning  # noqa F401
 from .workflowtask import WorkflowTaskStatusTypeV2  # noqa F401
 from .workflowtask import WorkflowTaskUpdateV2  # noqa F401

fractal_server/app/schemas/v2/task.py

@@ -90,7 +90,6 @@ class TaskReadV2(BaseModel):
     name: str
     type: Literal["parallel", "non_parallel", "compound"]
     source: str
-    owner: Optional[str]
     version: Optional[str]
 
     command_non_parallel: Optional[str]
@@ -105,6 +104,8 @@ class TaskReadV2(BaseModel):
     input_types: dict[str, bool]
     output_types: dict[str, bool]
 
+    taskgroupv2_id: Optional[int]
+
 
 class TaskUpdateV2(BaseModel):
 

fractal_server/app/schemas/v2/task_group.py

@@ -0,0 +1,23 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from .task import TaskReadV2
+
+
+class TaskGroupCreateV2(BaseModel):
+    active: bool = True
+
+
+class TaskGroupReadV2(BaseModel):
+
+    id: int
+    user_id: int
+    user_group_id: Optional[int] = None
+    active: bool
+    task_list: list[TaskReadV2]
+
+
+class TaskGroupUpdateV2(BaseModel):
+    user_group_id: Optional[int] = None
+    active: Optional[bool] = None

fractal_server/app/schemas/v2/workflow.py

@@ -11,6 +11,7 @@ from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskExportV2
 from .workflowtask import WorkflowTaskImportV2
 from .workflowtask import WorkflowTaskReadV2
+from .workflowtask import WorkflowTaskReadV2WithWarning
 
 
 class WorkflowCreateV2(BaseModel, extra=Extra.forbid):
@@ -35,6 +36,10 @@ class WorkflowReadV2(BaseModel):
     )
 
 
+class WorkflowReadV2WithWarnings(WorkflowReadV2):
+    task_list: list[WorkflowTaskReadV2WithWarning]
+
+
 class WorkflowUpdateV2(BaseModel):
 
     name: Optional[str]

fractal_server/app/schemas/v2/workflowtask.py

@@ -102,6 +102,10 @@ class WorkflowTaskReadV2(BaseModel):
     task: TaskReadV2
 
 
+class WorkflowTaskReadV2WithWarning(WorkflowTaskReadV2):
+    warning: Optional[str] = None
+
+
 class WorkflowTaskUpdateV2(BaseModel):
 
     meta_non_parallel: Optional[dict[str, Any]]

fractal_server/migrations/versions/7cf1baae8fb4_task_group_v2.py

@@ -0,0 +1,66 @@
+"""task group v2
+
+Revision ID: 7cf1baae8fb4
+Revises: da2cb2ac4255
+Create Date: 2024-10-01 12:31:46.792037
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "7cf1baae8fb4"
+down_revision = "da2cb2ac4255"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "taskgroupv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("user_group_id", sa.Integer(), nullable=True),
+        sa.Column("active", sa.Boolean(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["usergroup.id"],
+            name=op.f("fk_taskgroupv2_user_group_id_usergroup"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user_oauth.id"],
+            name=op.f("fk_taskgroupv2_user_id_user_oauth"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_taskgroupv2")),
+    )
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("taskgroupv2_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskv2_taskgroupv2_id_taskgroupv2"),
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskv2_taskgroupv2_id_taskgroupv2"),
+            type_="foreignkey",
+        )
+        batch_op.drop_column("taskgroupv2_id")
+
+    op.drop_table("taskgroupv2")
+    # ### end Alembic commands ###

fractal_server/tasks/v2/background_operations.py

@@ -17,11 +17,12 @@ from ..utils import get_collection_log
 from ..utils import get_collection_path
 from ..utils import get_log_path
 from ._TaskCollectPip import _TaskCollectPip
+from .database_operations import create_db_task_group_and_tasks
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import CollectionStateV2
-from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.schemas.v2 import CollectionStatusV2
 from fractal_server.app.schemas.v2 import TaskCreateV2
+from fractal_server.app.schemas.v2 import TaskGroupCreateV2
 from fractal_server.app.schemas.v2 import TaskReadV2
 from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.logger import get_logger
@@ -30,38 +31,6 @@ from fractal_server.logger import set_logger
 from fractal_server.tasks.v2._venv_pip import _create_venv_install_package_pip
 
 
-def _get_task_type(task: TaskCreateV2) -> str:
-    if task.command_non_parallel is None:
-        return "parallel"
-    elif task.command_parallel is None:
-        return "non_parallel"
-    else:
-        return "compound"
-
-
-def _insert_tasks(
-    task_list: list[TaskCreateV2],
-    db: DBSyncSession,
-    owner: Optional[str] = None,
-) -> list[TaskV2]:
-    """
-    Insert tasks into database
-    """
-
-    owner_dict = dict(owner=owner) if owner is not None else dict()
-
-    task_db_list = [
-        TaskV2(**t.dict(), **owner_dict, type=_get_task_type(t))
-        for t in task_list
-    ]
-    db.add_all(task_db_list)
-    db.commit()
-    for t in task_db_list:
-        db.refresh(t)
-    db.close()
-    return task_db_list
-
-
 def _set_collection_state_data_status(
     *,
     state_id: int,
@@ -232,9 +201,12 @@ def _check_task_files_exist(task_list: list[TaskCreateV2]) -> None:
 
 
 async def background_collect_pip(
+    *,
     state_id: int,
     venv_path: Path,
     task_pkg: _TaskCollectPip,
+    user_id: int,
+    user_group_id: Optional[int],
 ) -> None:
     """
     Setup venv, install package, collect tasks.
@@ -301,7 +273,15 @@ async def background_collect_pip(
                 python_bin=python_bin,
             )
             _check_task_files_exist(task_list=task_list)
-            tasks = _insert_tasks(task_list=task_list, db=db)
+
+            task_group = create_db_task_group_and_tasks(
+                task_list=task_list,
+                task_group_obj=TaskGroupCreateV2(),
+                user_id=user_id,
+                user_group_id=user_group_id,
+                db=db,
+            )
+
             logger.debug("collecting -  prepare tasks and update db " "- END")
             logger.debug("collecting - END")
 
@@ -310,7 +290,8 @@ async def background_collect_pip(
             collection_path = get_collection_path(venv_path)
             collection_state = db.get(CollectionStateV2, state_id)
             task_read_list = [
-                TaskReadV2(**task.model_dump()).dict() for task in tasks
+                TaskReadV2(**task.model_dump()).dict()
+                for task in task_group.task_list
             ]
             collection_state.data["task_list"] = task_read_list
             collection_state.data["log"] = get_collection_log(venv_path)