fractal-server 2.7.0a4__py3-none-any.whl → 2.7.0a6__py3-none-any.whl

fractal_server/app/routes/api/v2/workflow_import.py

@@ -0,0 +1,355 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import status
+from sqlmodel import or_
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import TaskV2
+from ....models.v2 import WorkflowV2
+from ....schemas.v2 import TaskImportV2Legacy
+from ....schemas.v2 import WorkflowImportV2
+from ....schemas.v2 import WorkflowReadV2WithWarnings
+from ....schemas.v2 import WorkflowTaskCreateV2
+from ._aux_functions import _check_workflow_exists
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _workflow_insert_task
+from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2.task import TaskGroupV2
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
+from fractal_server.app.schemas.v2.task import TaskImportV2
+from fractal_server.logger import set_logger
+
+router = APIRouter()
+
+
+logger = set_logger(__name__)
+
+
+async def _get_user_accessible_taskgroups(
+    *,
+    user_id: int,
+    db: AsyncSession,
+) -> list[TaskGroupV2]:
+    """
+    Retrieve list of task groups that the user has access to.
+    """
+    stm = select(TaskGroupV2).where(
+        or_(
+            TaskGroupV2.user_id == user_id,
+            TaskGroupV2.user_group_id.in_(
+                select(LinkUserGroup.group_id).where(
+                    LinkUserGroup.user_id == user_id
+                )
+            ),
+        )
+    )
+    res = await db.execute(stm)
+    accessible_task_groups = res.scalars().all()
+    logger.info(
+        f"Found {len(accessible_task_groups)} accessible "
+        f"task groups for {user_id=}."
+    )
+    return accessible_task_groups
+
+
+async def _get_task_by_source(
+    source: str,
+    task_groups_list: list[TaskGroupV2],
+) -> Optional[int]:
+    """
+    Find task with a given source.
+
+    Args:
+        task_import: Info on task to be imported.
+        user_id: ID of current user.
+        default_group_id: ID of default user group.
+        task_group_list: Current list of valid task groups.
+        db: Asynchronous db session
+
+    Return:
+        `id` of the matching task, or `None`.
+    """
+    task_id = next(
+        iter(
+            task.id
+            for task_group in task_groups_list
+            for task in task_group.task_list
+            if task.source == source
+        ),
+        None,
+    )
+    return task_id
+
+
+async def _disambiguate_task_groups(
+    *,
+    matching_task_groups: list[TaskGroupV2],
+    user_id: int,
+    db: AsyncSession,
+    default_group_id: int,
+) -> Optional[TaskV2]:
+    """
+    Disambiguate task groups based on ownership information.
+    """
+    # Highest priority: task groups created by user
+    for task_group in matching_task_groups:
+        if task_group.user_id == user_id:
+            logger.info(
+                "[_disambiguate_task_groups] "
+                f"Found task group {task_group.id} with {user_id=}, return."
+            )
+            return task_group
+    logger.info(
+        "[_disambiguate_task_groups] "
+        f"No task group found with {user_id=}, continue."
+    )
+
+    # Medium priority: task groups owned by default user group
+    for task_group in matching_task_groups:
+        if task_group.user_group_id == default_group_id:
+            logger.info(
+                "[_disambiguate_task_groups] "
+                f"Found task group {task_group.id} with user_group_id="
+                f"{default_group_id}, return."
+            )
+            return task_group
+    logger.info(
+        "[_disambiguate_task_groups] "
+        "No task group found with user_group_id="
+        f"{default_group_id}, continue."
+    )
+
+    # Lowest priority: task groups owned by other groups, sorted
+    # according to age of the user/usergroup link
+    logger.info(
+        "[_disambiguate_task_groups] "
+        "Now sorting remaining task groups by oldest-user-link."
+    )
+    user_group_ids = [
+        task_group.user_group_id for task_group in matching_task_groups
+    ]
+    stm = (
+        select(LinkUserGroup.group_id)
+        .where(LinkUserGroup.user_id == user_id)
+        .where(LinkUserGroup.group_id.in_(user_group_ids))
+        .order_by(LinkUserGroup.timestamp_created.asc())
+    )
+    res = await db.execute(stm)
+    oldest_user_group_id = res.scalars().first()
+    logger.info(
+        "[_disambiguate_task_groups] "
+        f"Result of sorting: {oldest_user_group_id=}."
+    )
+    task_group = next(
+        iter(
+            task_group
+            for task_group in matching_task_groups
+            if task_group.user_group_id == oldest_user_group_id
+        ),
+        None,
+    )
+    return task_group
+
+
+async def _get_task_by_taskimport(
+    *,
+    task_import: TaskImportV2,
+    task_groups_list: list[TaskGroupV2],
+    user_id: int,
+    default_group_id: int,
+    db: AsyncSession,
+) -> Optional[int]:
+    """
+    Find a task based on `task_import`.
+
+    Args:
+        task_import: Info on task to be imported.
+        user_id: ID of current user.
+        default_group_id: ID of default user group.
+        task_group_list: Current list of valid task groups.
+        db: Asynchronous db session
+
+    Return:
+        `id` of the matching task, or `None`.
+    """
+
+    logger.info(f"[_get_task_by_taskimport] START, {task_import=}")
+
+    # Filter by `pkg_name` and by presence of a task with given `name`.
+    matching_task_groups = [
+        task_group
+        for task_group in task_groups_list
+        if (
+            task_group.pkg_name == task_import.pkg_name
+            and task_import.name
+            in [task.name for task in task_group.task_list]
+        )
+    ]
+    if len(matching_task_groups) < 1:
+        logger.info(
+            "[_get_task_by_taskimport] "
+            f"No task group with {task_import.pkg_name=} "
+            f"and a task with {task_import.name=}."
+        )
+        return None
+
+    # Determine target `version`
+    # Note that task_import.version cannot be "", due to a validator
+    if task_import.version is None:
+        logger.info(
+            "[_get_task_by_taskimport] "
+            "No version requested, looking for latest."
+        )
+        latest_task = max(
+            matching_task_groups, key=lambda tg: tg.version or ""
+        )
+        version = latest_task.version
+        logger.info(
+            f"[_get_task_by_taskimport] Latest version set to {version}."
+        )
+    else:
+        version = task_import.version
+
+    # Filter task groups by version
+    final_matching_task_groups = list(
+        filter(lambda tg: tg.version == version, task_groups_list)
+    )
+
+    if len(final_matching_task_groups) < 1:
+        logger.info(
+            "[_get_task_by_taskimport] "
+            "No task group left after filtering by version."
+        )
+        return None
+    elif len(final_matching_task_groups) == 1:
+        final_task_group = final_matching_task_groups[0]
+        logger.info(
+            "[_get_task_by_taskimport] "
+            "Found a single task group, after filtering by version."
+        )
+    else:
+        logger.info(
+            "[_get_task_by_taskimport] "
+            "Found many task groups, after filtering by version."
+        )
+        final_task_group = await _disambiguate_task_groups(
+            matching_task_groups, user_id, db, default_group_id
+        )
+        if final_task_group is None:
+            logger.info(
+                "[_get_task_by_taskimport] Disambiguation returned None."
+            )
+            return None
+
+    # Find task with given name
+    task_id = next(
+        iter(
+            task.id
+            for task in final_task_group.task_list
+            if task.name == task_import.name
+        ),
+        None,
+    )
+
+    logger.info(f"[_get_task_by_taskimport] END, {task_import=}, {task_id=}.")
+
+    return task_id
+
+
+@router.post(
+    "/project/{project_id}/workflow/import/",
+    response_model=WorkflowReadV2WithWarnings,
+    status_code=status.HTTP_201_CREATED,
+)
+async def import_workflow(
+    project_id: int,
+    workflow_import: WorkflowImportV2,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> WorkflowReadV2WithWarnings:
+    """
+    Import an existing workflow into a project and create required objects.
+    """
+
+    # Preliminary checks
+    await _get_project_check_owner(
+        project_id=project_id,
+        user_id=user.id,
+        db=db,
+    )
+    await _check_workflow_exists(
+        name=workflow_import.name,
+        project_id=project_id,
+        db=db,
+    )
+
+    task_group_list = await _get_user_accessible_taskgroups(
+        user_id=user.id,
+        db=db,
+    )
+    default_group_id = await _get_default_usergroup_id(db)
+
+    list_wf_tasks = []
+    for wf_task in workflow_import.task_list:
+        task_import = wf_task.task
+        if isinstance(task_import, TaskImportV2Legacy):
+            task_id = await _get_task_by_source(
+                source=task_import.source,
+                task_groups_list=task_group_list,
+            )
+        else:
+            task_id = await _get_task_by_taskimport(
+                task_import=task_import,
+                user_id=user.id,
+                default_group_id=default_group_id,
+                task_groups_list=task_group_list,
+                db=db,
+            )
+        if task_id is None:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=f"Could not find a task matching with {wf_task.task}.",
+            )
+        new_wf_task = WorkflowTaskCreateV2(
+            **wf_task.dict(exclude_none=True, exclude={"task"})
+        )
+        list_wf_tasks.append(new_wf_task)
+
+    # Create new Workflow
+    db_workflow = WorkflowV2(
+        project_id=project_id,
+        **workflow_import.dict(exclude_none=True, exclude={"task_list"}),
+    )
+    db.add(db_workflow)
+    await db.commit()
+    await db.refresh(db_workflow)
+
+    # Insert task into the workflow
+    for new_wf_task in list_wf_tasks:
+        await _workflow_insert_task(
+            **new_wf_task.dict(),
+            workflow_id=db_workflow.id,
+            task_id=task_id,
+            db=db,
+        )
+
+    # Add warnings for non-active tasks (or non-accessible tasks,
+    # although that should never happen)
+    wftask_list_with_warnings = await _add_warnings_to_workflow_tasks(
+        wftask_list=db_workflow.task_list, user_id=user.id, db=db
+    )
+    workflow_data = dict(
+        **db_workflow.model_dump(),
+        project=db_workflow.project,
+        task_list=wftask_list_with_warnings,
+    )
+
+    return workflow_data

fractal_server/app/routes/api/v2/workflowtask.py

@@ -77,7 +77,6 @@ async def create_workflowtask(
     workflow_task = await _workflow_insert_task(
         workflow_id=workflow.id,
         task_id=task_id,
-        order=new_task.order,
         meta_non_parallel=new_task.meta_non_parallel,
         meta_parallel=new_task.meta_parallel,
         args_non_parallel=new_task.args_non_parallel,

fractal_server/app/routes/auth/group.py

@@ -19,10 +19,12 @@ from fractal_server.app.db import get_async_db
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserGroup
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.schemas.user_group import UserGroupCreate
 from fractal_server.app.schemas.user_group import UserGroupRead
 from fractal_server.app.schemas.user_group import UserGroupUpdate
+from fractal_server.app.schemas.user_settings import UserSettingsUpdate
 from fractal_server.app.security import FRACTAL_DEFAULT_GROUP_NAME
 from fractal_server.logger import set_logger
 
@@ -212,3 +214,28 @@ async def delete_single_group(
     await db.commit()
 
     return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router_group.patch("/group/{group_id}/user-settings/", status_code=200)
+async def patch_user_settings_bulk(
+    group_id: int,
+    settings_update: UserSettingsUpdate,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+):
+    await _usergroup_or_404(group_id, db)
+    res = await db.execute(
+        select(UserSettings)
+        .join(UserOAuth)
+        .where(LinkUserGroup.user_id == UserOAuth.id)
+        .where(LinkUserGroup.group_id == group_id)
+    )
+    settings_list = res.scalars().all()
+    update = settings_update.dict(exclude_unset=True)
+    for settings in settings_list:
+        for k, v in update.items():
+            setattr(settings, k, v)
+        db.add(settings)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_200_OK)

fractal_server/app/runner/v2/__init__.py

@@ -177,11 +177,13 @@ async def submit_workflow(
             return
 
         try:
-
             # Create WORKFLOW_DIR_LOCAL
-            original_umask = os.umask(0)
-            WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
-            os.umask(original_umask)
+            if FRACTAL_RUNNER_BACKEND == "slurm":
+                original_umask = os.umask(0)
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
+                os.umask(original_umask)
+            else:
+                WORKFLOW_DIR_LOCAL.mkdir(parents=True)
 
             # Define and create WORKFLOW_DIR_REMOTE
             if FRACTAL_RUNNER_BACKEND == "local":
@@ -214,15 +216,19 @@ async def submit_workflow(
                     order=order,
                     task_name=task_name,
                 )
-                original_umask = os.umask(0)
-                (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
-                os.umask(original_umask)
                 if FRACTAL_RUNNER_BACKEND == "slurm":
+                    # Create local subfolder (with 755) and remote one
+                    # (via `sudo -u`)
+                    original_umask = os.umask(0)
+                    (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+                    os.umask(original_umask)
                     _mkdir_as_user(
                         folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
                         user=slurm_user,
                     )
                 else:
+                    # Create local subfolder (with standard permission set)
+                    (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir()
                     logger.info("Skip remote-subfolder creation")
         except Exception as e:
             error_type = type(e).__name__

fractal_server/app/schemas/v2/__init__.py

@@ -20,6 +20,7 @@ from .project import ProjectUpdateV2  # noqa F401
 from .task import TaskCreateV2  # noqa F401
 from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
+from .task import TaskImportV2Legacy  # noqa F401
 from .task import TaskReadV2  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
 from .task_collection import CollectionStateReadV2  # noqa F401

fractal_server/app/schemas/v2/dumps.py

@@ -31,7 +31,6 @@ class TaskDumpV2(BaseModel):
     command_non_parallel: Optional[str]
     command_parallel: Optional[str]
     source: Optional[str] = None
-    owner: Optional[str]
     version: Optional[str]
 
     input_types: dict[str, bool]

fractal_server/app/schemas/v2/manifest.py

@@ -159,6 +159,19 @@ class ManifestV2(BaseModel):
                         )
         return values
 
+    @root_validator()
+    def _unique_task_names(cls, values):
+        task_list = values["task_list"]
+        task_list_names = [t.name for t in task_list]
+        if len(set(task_list_names)) != len(task_list_names):
+            raise ValueError(
+                (
+                    "Task names in manifest must be unique.\n",
+                    f"Given: {task_list_names}.",
+                )
+            )
+        return values
+
     @validator("manifest_version")
     def manifest_version_2(cls, value):
         if value != "2":

fractal_server/app/schemas/v2/task.py

@@ -21,7 +21,6 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
 
     command_non_parallel: Optional[str] = None
     command_parallel: Optional[str] = None
-    source: Optional[str] = None
 
     meta_non_parallel: Optional[dict[str, Any]] = None
     meta_parallel: Optional[dict[str, Any]] = None
@@ -64,7 +63,6 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
     _command_parallel = validator("command_parallel", allow_reuse=True)(
         valstr("command_parallel")
     )
-    _source = validator("source", allow_reuse=True)(valstr("source"))
     _version = validator("version", allow_reuse=True)(valstr("version"))
 
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
@@ -136,8 +134,6 @@ class TaskReadV2(BaseModel):
 
 class TaskUpdateV2(BaseModel, extra=Extra.forbid):
 
-    name: Optional[str] = None
-    version: Optional[str] = None
     command_parallel: Optional[str] = None
     command_non_parallel: Optional[str] = None
     input_types: Optional[dict[str, bool]] = None
@@ -155,10 +151,6 @@ class TaskUpdateV2(BaseModel, extra=Extra.forbid):
             raise ValueError
         return v
 
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-    _version = validator("version", allow_reuse=True)(
-        valstr("version", accept_none=True)
-    )
     _command_parallel = validator("command_parallel", allow_reuse=True)(
         valstr("command_parallel")
     )
@@ -191,11 +183,29 @@ class TaskUpdateV2(BaseModel, extra=Extra.forbid):
 
 class TaskImportV2(BaseModel, extra=Extra.forbid):
 
+    pkg_name: str
+    version: Optional[str] = None
+    name: str
+    _pkg_name = validator("pkg_name", allow_reuse=True)(valstr("pkg_name"))
+    _version = validator("version", allow_reuse=True)(
+        valstr("version", accept_none=True)
+    )
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class TaskImportV2Legacy(BaseModel):
     source: str
     _source = validator("source", allow_reuse=True)(valstr("source"))
 
 
 class TaskExportV2(BaseModel):
 
-    source: Optional[str] = None
-    _source = validator("source", allow_reuse=True)(valstr("source"))
+    pkg_name: str
+    version: Optional[str] = None
+    name: str
+
+    _pkg_name = validator("pkg_name", allow_reuse=True)(valstr("pkg_name"))
+    _version = validator("version", allow_reuse=True)(
+        valstr("version", accept_none=True)
+    )
+    _name = validator("name", allow_reuse=True)(valstr("name"))

fractal_server/app/schemas/v2/workflowtask.py

@@ -1,6 +1,7 @@
 from enum import Enum
 from typing import Any
 from typing import Optional
+from typing import Union
 
 from pydantic import BaseModel
 from pydantic import Extra
@@ -8,9 +9,9 @@ from pydantic import Field
 from pydantic import validator
 
 from .._validators import valdictkeys
-from .._validators import valint
 from .task import TaskExportV2
 from .task import TaskImportV2
+from .task import TaskImportV2Legacy
 from .task import TaskReadV2
 from fractal_server.images import Filters
 
@@ -42,7 +43,6 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    order: Optional[int]
     input_filters: Filters = Field(default_factory=Filters)
 
     # Validators
@@ -52,7 +52,6 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
         valdictkeys("meta_parallel")
     )
-    _order = validator("order", allow_reuse=True)(valint("order", min_val=0))
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
@@ -160,7 +159,7 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
 
     input_filters: Optional[Filters] = None
 
-    task: TaskImportV2
+    task: Union[TaskImportV2, TaskImportV2Legacy]
 
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
         valdictkeys("meta_non_parallel")