fractal-server 2.15.10a2__tar.gz → 2.15.10a3__tar.gz

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.10a2
+Version: 2.15.10a3
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.15.10a3/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.15.10a3"

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/admin/v2/task_group.py

@@ -1,7 +1,6 @@
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
-from fastapi import Response
 from fastapi import status
 from pydantic.types import AwareDatetime
 from sqlalchemy.sql.operators import is_
@@ -13,7 +12,6 @@ from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
-from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_superuser
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
@@ -161,33 +159,3 @@ async def patch_task_group(
     await db.commit()
     await db.refresh(task_group)
     return task_group
-
-
-@router.delete("/{task_group_id}/", status_code=204)
-async def delete_task_group(
-    task_group_id: int,
-    user: UserOAuth = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_async_db),
-):
-    task_group = await db.get(TaskGroupV2, task_group_id)
-    if task_group is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"TaskGroupV2 {task_group_id} not found",
-        )
-
-    stm = select(WorkflowTaskV2).where(
-        WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
-    )
-    res = await db.execute(stm)
-    workflow_tasks = res.scalars().all()
-    if workflow_tasks != []:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
-        )
-
-    await db.delete(task_group)
-    await db.commit()
-
-    return Response(status_code=status.HTTP_204_NO_CONTENT)

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/admin/v2/task_group_lifecycle.py

@@ -12,6 +12,9 @@ from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.routes.api.v2._aux_functions_task_lifecycle import (
     check_no_ongoing_activity,
 )
+from fractal_server.app.routes.api.v2._aux_functions_task_lifecycle import (
+    check_no_related_workflowtask,
+)
 from fractal_server.app.routes.api.v2._aux_functions_task_lifecycle import (
     check_no_submitted_job,
 )
@@ -31,8 +34,10 @@ from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.local import deactivate_local
+from fractal_server.tasks.v2.local import delete_local
 from fractal_server.tasks.v2.local import reactivate_local
 from fractal_server.tasks.v2.ssh import deactivate_ssh
+from fractal_server.tasks.v2.ssh import delete_ssh
 from fractal_server.tasks.v2.ssh import reactivate_ssh
 from fractal_server.utils import get_timestamp
 
@@ -258,3 +263,65 @@ async def reactivate_task_group(
     )
     response.status_code = status.HTTP_202_ACCEPTED
     return task_group_activity
+
+
+@router.post("/{task_group_id}/delete/", status_code=202)
+async def delete_task_group(
+    task_group_id: int,
+    background_tasks: BackgroundTasks,
+    response: Response,
+    superuser: UserOAuth = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+):
+    task_group = await _get_task_group_or_404(
+        task_group_id=task_group_id, db=db
+    )
+    await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
+    await check_no_submitted_job(task_group_id=task_group_id, db=db)
+    await check_no_related_workflowtask(task_group=task_group, db=db)
+
+    task_group_activity = TaskGroupActivityV2(
+        user_id=task_group.user_id,
+        taskgroupv2_id=task_group.id,
+        status=TaskGroupActivityStatusV2.PENDING,
+        action=TaskGroupActivityActionV2.DELETE,
+        pkg_name=task_group.pkg_name,
+        version=(task_group.version or "N/A"),
+        timestamp_started=get_timestamp(),
+    )
+    db.add(task_group_activity)
+    await db.commit()
+
+    settings = Inject(get_settings)
+    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+        # Validate user settings (backend-specific)
+        task_owner = await db.get(UserOAuth, task_group.user_id)
+        task_owner_settings = await validate_user_settings(
+            user=task_owner, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+        )
+        # Use appropriate FractalSSH object
+        ssh_config = SSHConfig(
+            user=task_owner_settings.ssh_username,
+            host=task_owner_settings.ssh_host,
+            key_path=task_owner_settings.ssh_private_key_path,
+        )
+
+        background_tasks.add_task(
+            delete_ssh,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+            ssh_config=ssh_config,
+            tasks_base_dir=task_owner_settings.ssh_tasks_dir,
+        )
+    else:
+        background_tasks.add_task(
+            delete_local,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+        )
+    logger.debug(
+        "Admin task group deletion endpoint: start deletion "
+        "and return task_group_activity"
+    )
+    response.status_code = status.HTTP_202_ACCEPTED
+    return task_group_activity

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py

@@ -8,6 +8,7 @@ from sqlmodel import select
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.models.v2 import TaskGroupActivityV2
+from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.models.v2 import WorkflowV2
@@ -203,3 +204,24 @@ async def check_no_submitted_job(
                 "submitted jobs use its tasks."
             ),
         )
+
+
+async def check_no_related_workflowtask(
+    *,
+    task_group: TaskGroupV2,
+    db: AsyncSession,
+) -> None:
+    """
+    Raises an HTTPException if any of the tasks in the TaskGroup are referenced
+    by an existing WorkflowTask.
+    """
+    stm = select(WorkflowTaskV2).where(
+        WorkflowTaskV2.task_id.in_([task.id for task in task_group.task_list])
+    )
+    res = await db.execute(stm)
+    bad_wftask = res.scalars().first()
+    if bad_wftask is not None:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"TaskV2 {bad_wftask.task_id} is still in use",
+        )

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -243,7 +243,7 @@ async def _get_collection_task_group_activity_status_message(
         )
     elif len(task_group_activity_list) == 1:
         msg = (
-            "\nNote:"
+            "\nNote: "
             "There exists another task-group collection "
             f"(activity ID={task_group_activity_list[0].id}) for "
             f"this task group (ID={task_group_id}), with status "

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/api/v2/task_group.py

@@ -3,7 +3,6 @@ import itertools
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
-from fastapi import Response
 from fastapi import status
 from packaging.version import InvalidVersion
 from packaging.version import parse
@@ -12,7 +11,6 @@ from pydantic.types import AwareDatetime
 from sqlmodel import or_
 from sqlmodel import select
 
-from ._aux_functions_task_lifecycle import check_no_ongoing_activity
 from ._aux_functions_tasks import _get_task_group_full_access
 from ._aux_functions_tasks import _get_task_group_read_access
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
@@ -23,7 +21,6 @@ from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
-from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import _get_default_usergroup_id
 from fractal_server.app.routes.auth._aux_auth import (
@@ -199,41 +196,6 @@ async def get_task_group(
     return task_group
 
 
-@router.delete("/{task_group_id}/", status_code=204)
-async def delete_task_group(
-    task_group_id: int,
-    user: UserOAuth = Depends(current_active_user),
-    db: AsyncSession = Depends(get_async_db),
-):
-    """
-    Delete single TaskGroup
-    """
-
-    task_group = await _get_task_group_full_access(
-        task_group_id=task_group_id,
-        user_id=user.id,
-        db=db,
-    )
-
-    await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
-
-    stm = select(WorkflowTaskV2).where(
-        WorkflowTaskV2.task_id.in_({task.id for task in task_group.task_list})
-    )
-    res = await db.execute(stm)
-    workflow_tasks = res.scalars().all()
-    if workflow_tasks != []:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
-        )
-
-    await db.delete(task_group)
-    await db.commit()
-
-    return Response(status_code=status.HTTP_204_NO_CONTENT)
-
-
 @router.patch("/{task_group_id}/", response_model=TaskGroupReadV2)
 async def patch_task_group(
     task_group_id: int,

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/routes/api/v2/task_group_lifecycle.py

@@ -7,6 +7,7 @@ from fastapi import status
 
 from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions_task_lifecycle import check_no_ongoing_activity
+from ._aux_functions_task_lifecycle import check_no_related_workflowtask
 from ._aux_functions_task_lifecycle import check_no_submitted_job
 from ._aux_functions_tasks import _get_task_group_full_access
 from fractal_server.app.db import AsyncSession
@@ -25,10 +26,12 @@ from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.local import deactivate_local
 from fractal_server.tasks.v2.local import deactivate_local_pixi
+from fractal_server.tasks.v2.local import delete_local
 from fractal_server.tasks.v2.local import reactivate_local
 from fractal_server.tasks.v2.local import reactivate_local_pixi
 from fractal_server.tasks.v2.ssh import deactivate_ssh
 from fractal_server.tasks.v2.ssh import deactivate_ssh_pixi
+from fractal_server.tasks.v2.ssh import delete_ssh
 from fractal_server.tasks.v2.ssh import reactivate_ssh
 from fractal_server.tasks.v2.ssh import reactivate_ssh_pixi
 from fractal_server.utils import get_timestamp
@@ -281,3 +284,69 @@ async def reactivate_task_group(
     )
     response.status_code = status.HTTP_202_ACCEPTED
     return task_group_activity
+
+
+@router.post(
+    "/{task_group_id}/delete/",
+    status_code=202,
+)
+async def delete_task_group(
+    task_group_id: int,
+    background_tasks: BackgroundTasks,
+    response: Response,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskGroupActivityV2Read:
+    """
+    Deletion of task-group from db and file system
+    """
+
+    task_group = await _get_task_group_full_access(
+        task_group_id=task_group_id,
+        user_id=user.id,
+        db=db,
+    )
+    await check_no_ongoing_activity(task_group_id=task_group_id, db=db)
+    await check_no_related_workflowtask(task_group=task_group, db=db)
+
+    task_group_activity = TaskGroupActivityV2(
+        user_id=task_group.user_id,
+        taskgroupv2_id=task_group.id,
+        status=TaskGroupActivityStatusV2.PENDING,
+        action=TaskGroupActivityActionV2.DELETE,
+        pkg_name=task_group.pkg_name,
+        version=(task_group.version or "N/A"),
+        timestamp_started=get_timestamp(),
+    )
+    db.add(task_group_activity)
+    await db.commit()
+
+    settings = Inject(get_settings)
+    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+        # Validate user settings (backend-specific)
+        user_settings = await validate_user_settings(
+            user=user, backend=settings.FRACTAL_RUNNER_BACKEND, db=db
+        )
+        # User appropriate FractalSSH object
+        ssh_config = SSHConfig(
+            user=user_settings.ssh_username,
+            host=user_settings.ssh_host,
+            key_path=user_settings.ssh_private_key_path,
+        )
+
+        background_tasks.add_task(
+            delete_ssh,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+            ssh_config=ssh_config,
+            tasks_base_dir=user_settings.ssh_tasks_dir,
+        )
+    else:
+        background_tasks.add_task(
+            delete_local,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
+        )
+
+    response.status_code = status.HTTP_202_ACCEPTED
+    return task_group_activity

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/runner/v2/runner.py

@@ -194,6 +194,16 @@ def execute_tasks_v2(
             db.refresh(history_run)
             history_run_id = history_run.id
 
+            # Refresh `job.executor_error_log`, to avoid a spurious value left
+            # over from a previous task
+            job_db = db.get(JobV2, job_id)
+            job_db.executor_error_log = None
+            logger.debug(
+                f"Resetting `JobV2[{job_id}].executor_error_log` to None."
+            )
+            db.merge(job_db)
+            db.commit()
+
         # TASK EXECUTION (V2)
         try:
             if task.type in [

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/runner/v2/submit_workflow.py

@@ -244,6 +244,7 @@ def submit_workflow(
         logger.debug(f"job.first_task_index: {job.first_task_index}")
         logger.debug(f"job.last_task_index: {job.last_task_index}")
         logger.debug(f'START workflow "{workflow.name}"')
+        job_working_dir = job.working_dir
 
     try:
         if FRACTAL_RUNNER_BACKEND == "local":
@@ -267,19 +268,6 @@ def submit_workflow(
                 f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}"
             )
 
-        # "The Session.close() method does not prevent the Session from being
-        # used again. The Session itself does not actually have a distinct
-        # “closed” state; it merely means the Session will release all database
-        # connections and ORM objects."
-        # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close).
-        #
-        # We close the session before the (possibly long) process_workflow
-        # call, to make sure all DB connections are released. The reason why we
-        # are not using a context manager within the try block is that we also
-        # need access to db_sync in the except branches.
-        db_sync = next(DB.get_sync_db())
-        db_sync.close()
-
         process_workflow(
             workflow=workflow,
             dataset=dataset,
@@ -303,47 +291,48 @@ def submit_workflow(
         logger.debug(f'END workflow "{workflow.name}"')
 
         # Update job DB entry
-        job = db_sync.get(JobV2, job_id)  # refetch, in case it was updated
-        job.status = JobStatusTypeV2.DONE
-        job.end_timestamp = get_timestamp()
-        with log_file_path.open("r") as f:
-            logs = f.read()
-        job.log = logs
-        db_sync.merge(job)
-        db_sync.commit()
+        with next(DB.get_sync_db()) as db_sync:
+            job = db_sync.get(JobV2, job_id)
+            job.status = JobStatusTypeV2.DONE
+            job.end_timestamp = get_timestamp()
+            with log_file_path.open("r") as f:
+                logs = f.read()
+            job.log = logs
+            db_sync.merge(job)
+            db_sync.commit()
 
     except JobExecutionError as e:
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
-
-        fail_job(
-            db=db_sync,
-            job=job,
-            log_msg=(
-                f"JOB ERROR in Fractal job {job.id}:\n"
-                f"TRACEBACK:\n{e.assemble_error()}"
-            ),
-            logger_name=logger_name,
-        )
+        with next(DB.get_sync_db()) as db_sync:
+            job = db_sync.get(JobV2, job_id)
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"JOB ERROR in Fractal job {job.id}:\n"
+                    f"TRACEBACK:\n{e.assemble_error()}"
+                ),
+                logger_name=logger_name,
+            )
 
     except Exception:
         logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
         logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
 
         current_traceback = traceback.format_exc()
-        fail_job(
-            db=db_sync,
-            job=job,
-            log_msg=(
-                f"UNKNOWN ERROR in Fractal job {job.id}\n"
-                f"TRACEBACK:\n{current_traceback}"
-            ),
-            logger_name=logger_name,
-        )
+        with next(DB.get_sync_db()) as db_sync:
+            job = db_sync.get(JobV2, job_id)
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"UNKNOWN ERROR in Fractal job {job.id}\n"
+                    f"TRACEBACK:\n{current_traceback}"
+                ),
+                logger_name=logger_name,
+            )
 
     finally:
         reset_logger_handlers(logger)
-        # refetch, in case it is not already available
-        job = db_sync.get(JobV2, job_id)
-        db_sync.close()
-        _zip_folder_to_file_and_remove(folder=job.working_dir)
+        _zip_folder_to_file_and_remove(folder=job_working_dir)

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/app/schemas/v2/task_group.py

@@ -31,6 +31,7 @@ class TaskGroupActivityActionV2(StrEnum):
     COLLECT = "collect"
     DEACTIVATE = "deactivate"
     REACTIVATE = "reactivate"
+    DELETE = "delete"
 
 
 class TaskGroupCreateV2(BaseModel):

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/tasks/v2/local/__init__.py

@@ -2,5 +2,6 @@ from .collect import collect_local  # noqa
 from .collect_pixi import collect_local_pixi  # noqa
 from .deactivate import deactivate_local  # noqa
 from .deactivate_pixi import deactivate_local_pixi  # noqa
+from .delete import delete_local  # noqa
 from .reactivate import reactivate_local  # noqa
 from .reactivate_pixi import reactivate_local_pixi  # noqa

fractal_server-2.15.10a3/fractal_server/tasks/v2/local/delete.py

@@ -0,0 +1,76 @@
+import shutil
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
+from fractal_server.logger import reset_logger_handlers
+from fractal_server.logger import set_logger
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import fail_and_cleanup
+from fractal_server.tasks.v2.utils_background import (
+    get_activity_and_task_group,
+)
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.utils import get_timestamp
+
+
+def delete_local(
+    *,
+    task_group_activity_id: int,
+    task_group_id: int,
+) -> None:
+    LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
+
+    with TemporaryDirectory() as tmpdir:
+        log_file_path = get_log_path(Path(tmpdir))
+
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+        logger.debug("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
+
+            try:
+                activity.status = TaskGroupActivityStatusV2.ONGOING
+                activity.log = get_current_log(log_file_path)
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                db.commit()
+                db.delete(task_group)
+                logger.debug("Task group removed from database.")
+
+                if task_group.origin != TaskGroupV2OriginEnum.OTHER:
+                    logger.debug(f"Removing {task_group.path=}.")
+                    shutil.rmtree(task_group.path)
+                    logger.debug(f"{task_group.path=} removed.")
+
+                activity.status = TaskGroupActivityStatusV2.OK
+                activity.log = get_current_log(log_file_path)
+                activity.timestamp_ended = get_timestamp()
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                logger.debug("END")
+
+            except Exception as delete_e:
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=delete_e,
+                    db=db,
+                )
+            finally:
+                reset_logger_handlers(logger)

{fractal_server-2.15.10a2 → fractal_server-2.15.10a3}/fractal_server/tasks/v2/ssh/__init__.py

@@ -2,5 +2,6 @@ from .collect import collect_ssh  # noqa
 from .collect_pixi import collect_ssh_pixi  # noqa
 from .deactivate import deactivate_ssh  # noqa
 from .deactivate_pixi import deactivate_ssh_pixi  # noqa
+from .delete import delete_ssh  # noqa
 from .reactivate import reactivate_ssh  # noqa
 from .reactivate_pixi import reactivate_ssh_pixi  # noqa