fractal-server 2.7.0a4__tar.gz → 2.7.0a6__tar.gz

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.7.0a4
+Version: 2.7.0a6
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.7.0a6/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.7.0a6"

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/admin/v2/task.py

@@ -29,7 +29,6 @@ class TaskV2Minimal(BaseModel):
     command_non_parallel: Optional[str] = None
     command_parallel: Optional[str]
     source: Optional[str] = None
-    owner: Optional[str] = None
     version: Optional[str] = None
 
 
@@ -60,7 +59,6 @@ async def query_tasks(
     source: Optional[str] = None,
     version: Optional[str] = None,
     name: Optional[str] = None,
-    owner: Optional[str] = None,
     max_number_of_results: int = 25,
     user: UserOAuth = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
@@ -75,7 +73,6 @@ async def query_tasks(
             `task.source`.
         version: If not `None`, query for matching `task.version`.
         name: If not `None`, query for contained case insensitive `task.name`.
-        owner: If not `None`, query for matching `task.owner`.
         max_number_of_results: The maximum length of the response.
     """
 
@@ -89,8 +86,6 @@ async def query_tasks(
         stm = stm.where(TaskV2.version == version)
     if name is not None:
         stm = stm.where(TaskV2.name.icontains(name))
-    if owner is not None:
-        stm = stm.where(TaskV2.owner == owner)
 
     res = await db.execute(stm)
     task_list = res.scalars().all()

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v1/task_collection.py

@@ -27,7 +27,7 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
 from fractal_server.string_tools import slugify_task_name_for_source_v1
-from fractal_server.tasks.utils import get_collection_log
+from fractal_server.tasks.utils import get_collection_log_v1
 from fractal_server.tasks.v1._TaskCollectPip import _TaskCollectPip
 from fractal_server.tasks.v1.background_operations import (
     background_collect_pip,
@@ -232,7 +232,7 @@ async def check_collection_status(
     # In some cases (i.e. a successful or ongoing task collection), data.log is
     # not set; if so, we collect the current logs
     if verbose and not data.log:
-        data.log = get_collection_log(data.venv_path)
+        data.log = get_collection_log_v1(data.venv_path)
         state.data = data.sanitised_dict()
     close_logger(logger)
     await db.close()

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/__init__.py

@@ -14,6 +14,7 @@ from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
 from .task_group import router as task_group_router_v2
 from .workflow import router as workflow_router_v2
+from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
 from fractal_server.config import get_settings
 from fractal_server.syringe import Inject
@@ -42,5 +43,8 @@ router_api_v2.include_router(
     task_group_router_v2, prefix="/task-group", tags=["V2 TaskGroup"]
 )
 router_api_v2.include_router(workflow_router_v2, tags=["V2 Workflow"])
+router_api_v2.include_router(
+    workflow_import_router_v2, tags=["V2 Workflow Import"]
+)
 router_api_v2.include_router(workflowtask_router_v2, tags=["V2 WorkflowTask"])
 router_api_v2.include_router(status_router_v2, tags=["V2 Status"])

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/_aux_functions.py

@@ -332,7 +332,6 @@ async def _workflow_insert_task(
     *,
     workflow_id: int,
     task_id: int,
-    order: Optional[int] = None,
     meta_parallel: Optional[dict[str, Any]] = None,
     meta_non_parallel: Optional[dict[str, Any]] = None,
     args_non_parallel: Optional[dict[str, Any]] = None,
@@ -347,7 +346,6 @@ async def _workflow_insert_task(
         workflow_id:
         task_id:
 
-        order:
         meta_parallel:
         meta_non_parallel:
         args_non_parallel:
@@ -359,9 +357,6 @@ async def _workflow_insert_task(
     if db_workflow is None:
         raise ValueError(f"Workflow {workflow_id} does not exist")
 
-    if order is None:
-        order = len(db_workflow.task_list)
-
     # Get task from db
     db_task = await db.get(TaskV2, task_id)
     if db_task is None:
@@ -397,8 +392,7 @@ async def _workflow_insert_task(
         meta_non_parallel=final_meta_non_parallel,
         **input_filters_kwarg,
     )
-    db_workflow.task_list.insert(order, wf_task)
-    db_workflow.task_list.reorder()  # type: ignore
+    db_workflow.task_list.append(wf_task)
     flag_modified(db_workflow, "task_list")
     await db.commit()
 

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -9,13 +9,21 @@ from fastapi import HTTPException
 from fastapi import status
 from sqlmodel import select
 
-from ....db import AsyncSession
-from ....models import LinkUserGroup
-from ....models.v2 import TaskGroupV2
-from ....models.v2 import TaskV2
-from ....models.v2 import WorkflowTaskV2
-from ...auth._aux_auth import _get_default_usergroup_id
-from ...auth._aux_auth import _verify_user_belongs_to_group
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.models import LinkUserGroup
+from fractal_server.app.models import UserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.models.v2 import TaskV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
+from fractal_server.app.routes.auth._aux_auth import (
+    _verify_user_belongs_to_group,
+)
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
 
 
 async def _get_task_group_or_404(
@@ -211,6 +219,33 @@ async def _get_valid_user_group_id(
     return user_group_id
 
 
+async def _get_collection_status_message(
+    task_group: TaskGroupV2, db: AsyncSession
+) -> str:
+    res = await db.execute(
+        select(CollectionStateV2).where(
+            CollectionStateV2.taskgroupv2_id == task_group.id
+        )
+    )
+    states = res.scalars().all()
+    if len(states) > 1:
+        msg = (
+            "Expected one CollectionStateV2 associated to TaskGroup "
+            f"{task_group.id}, found {len(states)} "
+            f"(IDs: {[state.id for state in states]}).\n"
+            "Warning: this should have not happened, please contact an admin."
+        )
+    elif len(states) == 1:
+        msg = (
+            f"\nThere exists a task collection state (ID={states[0].id}) for "
+            f"this task group (ID={task_group.id}), with status "
+            f"{states[0].data.get('status')}."
+        )
+    else:
+        msg = ""
+    return msg
+
+
 async def _verify_non_duplication_user_constraint(
     db: AsyncSession,
     user_id: int,
@@ -226,11 +261,24 @@ async def _verify_non_duplication_user_constraint(
     res = await db.execute(stm)
     duplicate = res.scalars().all()
     if duplicate:
+        user = await db.get(UserOAuth, user_id)
+        if len(duplicate) > 1:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Invalid state:\n"
+                    f"User '{user.email}' already owns {len(duplicate)} task "
+                    f"groups with name='{pkg_name}' and {version=} "
+                    f"(IDs: {[group.id for group in duplicate]}).\n"
+                    "This should have not happened: please contact an admin."
+                ),
+            )
+        state_msg = await _get_collection_status_message(duplicate[0], db)
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
-                "There is already a TaskGroupV2 with "
-                f"({pkg_name=}, {version=}, {user_id=})."
+                f"User '{user.email}' already owns a task group "
+                f"with name='{pkg_name}' and {version=}.{state_msg}"
             ),
         )
 
@@ -253,11 +301,24 @@ async def _verify_non_duplication_group_constraint(
     res = await db.execute(stm)
     duplicate = res.scalars().all()
     if duplicate:
+        user_group = await db.get(UserGroup, user_group_id)
+        if len(duplicate) > 1:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Invalid state:\n"
+                    f"UserGroup '{user_group.name}' already owns "
+                    f"{len(duplicate)} task groups with name='{pkg_name}' and "
+                    f"{version=} (IDs: {[group.id for group in duplicate]}).\n"
+                    "This should have not happened: please contact an admin."
+                ),
+            )
+        state_msg = await _get_collection_status_message(duplicate[0], db)
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
-                "There is already a TaskGroupV2 with "
-                f"({pkg_name=}, {version=}, {user_group_id=})."
+                f"UserGroup {user_group.name} already owns a task group "
+                f"with {pkg_name=} and {version=}.{state_msg}"
             ),
         )
 

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/task_collection.py

@@ -31,7 +31,7 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
 from fractal_server.tasks.utils import _normalize_package_name
-from fractal_server.tasks.utils import get_collection_log
+from fractal_server.tasks.utils import get_collection_log_v2
 from fractal_server.tasks.v2.background_operations import (
     background_collect_pip,
 )
@@ -70,33 +70,40 @@ async def collect_tasks_pip(
     # Get settings
     settings = Inject(get_settings)
 
+    # Initialize task-group attributes
+    task_group_attrs = dict(user_id=user.id)
+
     # Set/check python version
     if task_collect.python_version is None:
-        task_collect.python_version = (
-            settings.FRACTAL_TASKS_PYTHON_DEFAULT_VERSION
-        )
+        task_group_attrs[
+            "python_version"
+        ] = settings.FRACTAL_TASKS_PYTHON_DEFAULT_VERSION
+    else:
+        task_group_attrs["python_version"] = task_collect.python_version
     try:
-        get_python_interpreter_v2(python_version=task_collect.python_version)
+        get_python_interpreter_v2(
+            python_version=task_group_attrs["python_version"]
+        )
     except ValueError:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
-                f"Python version {task_collect.python_version} is "
+                f"Python version {task_group_attrs['python_version']} is "
                 "not available for Fractal task collection."
             ),
         )
 
-    # Populate task-group attributes
-    task_group_attrs = dict(
-        user_id=user.id,
-        python_version=task_collect.python_version,
-    )
+    # Set pip_extras
     if task_collect.package_extras is not None:
         task_group_attrs["pip_extras"] = task_collect.package_extras
+
+    # Set pinned_package_versions
     if task_collect.pinned_package_versions is not None:
         task_group_attrs[
             "pinned_package_versions"
         ] = task_collect.pinned_package_versions
+
+    # Set pkg_name, version, origin and wheel_path
     if task_collect.package.endswith(".whl"):
         try:
             task_group_attrs["wheel_path"] = task_collect.package
@@ -119,19 +126,11 @@ async def collect_tasks_pip(
         pkg_name = task_collect.package
         task_group_attrs["pkg_name"] = _normalize_package_name(pkg_name)
         task_group_attrs["origin"] = "pypi"
-        if task_collect.package_version is None:
-            latest_version = await get_package_version_from_pypi(
-                task_collect.package
-            )
-            task_group_attrs["version"] = latest_version
-            task_collect.package_version = latest_version
-        else:
-            task_group_attrs["version"] = task_collect.package_version
-
-    # Validate user settings (backend-specific)
-    user_settings = await validate_user_settings(
-        user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
-    )
+        latest_version = await get_package_version_from_pypi(
+            task_collect.package,
+            task_collect.package_version,
+        )
+        task_group_attrs["version"] = latest_version
 
     # Validate query parameters related to user-group ownership
     user_group_id = await _get_valid_user_group_id(
@@ -140,9 +139,16 @@ async def collect_tasks_pip(
         user_id=user.id,
         db=db,
     )
+
+    # Set user_group_id
     task_group_attrs["user_group_id"] = user_group_id
 
-    # Construct task_group.path
+    # Validate user settings (backend-specific)
+    user_settings = await validate_user_settings(
+        user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+    )
+
+    # Set path and venv_path
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         base_tasks_path = user_settings.ssh_tasks_dir
     else:
@@ -165,6 +171,8 @@ async def collect_tasks_pip(
             detail=f"Invalid task-group object. Original error: {e}",
         )
 
+    # Database checks
+
     # Verify non-duplication constraints
     await _verify_non_duplication_user_constraint(
         user_id=user.id,
@@ -191,15 +199,18 @@ async def collect_tasks_pip(
             ),
         )
 
-    # Verify that folder does not exist (for local collection)
+    # On-disk checks
+
     if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
+
+        # Verify that folder does not exist (for local collection)
         if Path(task_group_path).exists():
             raise HTTPException(
                 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                 detail=f"{task_group_path} already exists.",
             )
 
-    if settings.FRACTAL_RUNNER_BACKEND != "slurm_ssh":
+        # Verify that wheel file exists
         wheel_path = task_group_attrs.get("wheel_path", None)
         if wheel_path is not None:
             if not Path(wheel_path).exists():
@@ -216,13 +227,15 @@ async def collect_tasks_pip(
     db.expunge(task_group)
 
     # All checks are OK, proceed with task collection
-    collection_status = dict(
+    collection_state_data = dict(
         status=CollectionStatusV2.PENDING,
-        venv_path=task_group_attrs["venv_path"],
-        package=task_collect.package,
+        package=task_group.pkg_name,
+        version=task_group.version,
+        path=task_group.path,
+        venv_path=task_group.venv_path,
     )
     state = CollectionStateV2(
-        data=collection_status, taskgroupv2_id=task_group.id
+        data=collection_state_data, taskgroupv2_id=task_group.id
     )
     db.add(state)
     await db.commit()
@@ -270,7 +283,7 @@ async def collect_tasks_pip(
     reset_logger_handlers(logger)
     info = (
         "Collecting tasks in the background. "
-        f"GET /task/collect/{state.id} to query collection status"
+        f"GET /task/collect/{state.id}/ to query collection status"
     )
     state.data["info"] = info
     response.status_code = status.HTTP_201_CREATED
@@ -306,20 +319,17 @@ async def check_collection_status(
     else:
         # Non-SSH mode
         # In some cases (i.e. a successful or ongoing task collection),
-        # state.data.log is not set; if so, we collect the current logs.
+        # state.data["log"] is not set; if so, we collect the current logs.
         if verbose and not state.data.get("log"):
-            if "venv_path" not in state.data.keys():
+            if "path" not in state.data.keys():
                 await db.close()
                 raise HTTPException(
                     status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
                     detail=(
-                        f"No 'venv_path' in CollectionStateV2[{state_id}].data"
+                        f"No 'path' in CollectionStateV2[{state_id}].data"
                     ),
                 )
-            state.data["log"] = get_collection_log(
-                Path(state.data["venv_path"])
-            )
-            state.data["venv_path"] = str(state.data["venv_path"])
+            state.data["log"] = get_collection_log_v2(Path(state.data["path"]))
 
     reset_logger_handlers(logger)
     await db.close()

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/task_group.py

@@ -143,12 +143,16 @@ async def patch_task_group(
         user_id=user.id,
         db=db,
     )
-    await _verify_non_duplication_group_constraint(
-        db=db,
-        pkg_name=task_group.pkg_name,
-        version=task_group.version,
-        user_group_id=task_group_update.user_group_id,
-    )
+    if (
+        "user_group_id" in task_group_update.dict(exclude_unset=True)
+        and task_group_update.user_group_id != task_group.user_group_id
+    ):
+        await _verify_non_duplication_group_constraint(
+            db=db,
+            pkg_name=task_group.pkg_name,
+            version=task_group.version,
+            user_group_id=task_group_update.user_group_id,
+        )
     for key, value in task_group_update.dict(exclude_unset=True).items():
         if (key == "user_group_id") and (value is not None):
             await _verify_user_belongs_to_group(

{fractal_server-2.7.0a4 → fractal_server-2.7.0a6}/fractal_server/app/routes/api/v2/workflow.py

@@ -11,25 +11,21 @@ from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
-from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import WorkflowCreateV2
 from ....schemas.v2 import WorkflowExportV2
-from ....schemas.v2 import WorkflowImportV2
 from ....schemas.v2 import WorkflowReadV2
 from ....schemas.v2 import WorkflowReadV2WithWarnings
-from ....schemas.v2 import WorkflowTaskCreateV2
 from ....schemas.v2 import WorkflowUpdateV2
 from ._aux_functions import _check_workflow_exists
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
-from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2.task import TaskGroupV2
 from fractal_server.app.routes.auth import current_active_user
 
-
 router = APIRouter()
 
 
@@ -256,85 +252,21 @@ async def export_worfklow(
         user_id=user.id,
         db=db,
     )
-    return workflow
-
-
-@router.post(
-    "/project/{project_id}/workflow/import/",
-    response_model=WorkflowReadV2,
-    status_code=status.HTTP_201_CREATED,
-)
-async def import_workflow(
-    project_id: int,
-    workflow: WorkflowImportV2,
-    user: UserOAuth = Depends(current_active_user),
-    db: AsyncSession = Depends(get_async_db),
-) -> Optional[WorkflowReadV2]:
-    """
-    Import an existing workflow into a project
-
-    Also create all required objects (i.e. Workflow and WorkflowTask's) along
-    the way.
-    """
-
-    # Preliminary checks
-    await _get_project_check_owner(
-        project_id=project_id,
-        user_id=user.id,
-        db=db,
-    )
-
-    await _check_workflow_exists(
-        name=workflow.name, project_id=project_id, db=db
-    )
-
-    # Check that all required tasks are available
-    source_to_id = {}
-
-    for wf_task in workflow.task_list:
-
-        source = wf_task.task.source
-        if source not in source_to_id.keys():
-            stm = select(TaskV2).where(TaskV2.source == source)
-            tasks_by_source = (await db.execute(stm)).scalars().all()
-            if len(tasks_by_source) != 1:
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=(
-                        f"Found {len(tasks_by_source)} tasks "
-                        f"with {source=}."
-                    ),
-                )
-            source_to_id[source] = tasks_by_source[0].id
-
-    # Create new Workflow (with empty task_list)
-    db_workflow = WorkflowV2(
-        project_id=project_id,
-        **workflow.dict(exclude_none=True, exclude={"task_list"}),
-    )
-    db.add(db_workflow)
-    await db.commit()
-    await db.refresh(db_workflow)
-
-    # Insert tasks
-
-    for wf_task in workflow.task_list:
-        source = wf_task.task.source
-        task_id = source_to_id[source]
-
-        new_wf_task = WorkflowTaskCreateV2(
-            **wf_task.dict(exclude_none=True, exclude={"task"})
-        )
-        # Insert task
-        await _workflow_insert_task(
-            **new_wf_task.dict(),
-            workflow_id=db_workflow.id,
-            task_id=task_id,
-            db=db,
+    wf_task_list = []
+    for wftask in workflow.task_list:
+        task_group = await db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
+        wf_task_list.append(wftask.dict())
+        wf_task_list[-1]["task"] = dict(
+            pkg_name=task_group.pkg_name,
+            version=task_group.version,
+            name=wftask.task.name,
         )
 
-    await db.close()
-    return db_workflow
+    wf = WorkflowExportV2(
+        **workflow.model_dump(),
+        task_list=wf_task_list,
+    )
+    return wf
 
 
 @router.get("/workflow/", response_model=list[WorkflowReadV2])