fractal-server 2.8.0__py3-none-any.whl → 2.9.0__py3-none-any.whl

fractal_server/app/routes/api/v2/submit.py

@@ -1,6 +1,5 @@
+import json
 import os
-from datetime import datetime
-from datetime import timezone
 from pathlib import Path
 from typing import Optional
 
@@ -15,7 +14,6 @@ from sqlmodel import select
 from .....config import get_settings
 from .....logger import set_logger
 from .....syringe import Inject
-from .....utils import get_timestamp
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import JobV2
@@ -30,6 +28,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
+from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
@@ -37,10 +36,6 @@ from fractal_server.app.routes.api.v2._aux_functions_tasks import (
 from fractal_server.app.routes.auth import current_active_verified_user
 
 
-def _encode_as_utc(dt: datetime):
-    return dt.replace(tzinfo=timezone.utc).isoformat()
-
-
 router = APIRouter()
 logger = set_logger(__name__)
 
@@ -112,22 +107,23 @@ async def apply_workflow(
         )
 
     # Check that tasks have read-access and are `active`
+    used_task_group_ids = set()
     for wftask in workflow.task_list[
         first_task_index : last_task_index + 1  # noqa: E203
     ]:
-        await _get_task_read_access(
+        task = await _get_task_read_access(
             user_id=user.id,
             task_id=wftask.task_id,
             require_active=True,
             db=db,
         )
+        used_task_group_ids.add(task.taskgroupv2_id)
 
     # Validate user settings
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     user_settings = await validate_user_settings(
         user=user, backend=FRACTAL_RUNNER_BACKEND, db=db
     )
-
     # Check that no other job with the same dataset_id is SUBMITTED
     stm = (
         select(JobV2)
@@ -163,20 +159,9 @@ async def apply_workflow(
         dataset_id=dataset_id,
         workflow_id=workflow_id,
         user_email=user.email,
-        dataset_dump=dict(
-            **dataset.model_dump(
-                exclude={"images", "history", "timestamp_created"}
-            ),
-            timestamp_created=_encode_as_utc(dataset.timestamp_created),
-        ),
-        workflow_dump=dict(
-            **workflow.model_dump(exclude={"task_list", "timestamp_created"}),
-            timestamp_created=_encode_as_utc(workflow.timestamp_created),
-        ),
-        project_dump=dict(
-            **project.model_dump(exclude={"user_list", "timestamp_created"}),
-            timestamp_created=_encode_as_utc(project.timestamp_created),
-        ),
+        dataset_dump=json.loads(dataset.json(exclude={"images", "history"})),
+        workflow_dump=json.loads(workflow.json(exclude={"task_list"})),
+        project_dump=json.loads(project.json(exclude={"user_list"})),
         **job_create.dict(),
     )
 
@@ -184,8 +169,18 @@ async def apply_workflow(
     await db.commit()
     await db.refresh(job)
 
+    # Update TaskGroupV2.timestamp_last_used
+    res = await db.execute(
+        select(TaskGroupV2).where(TaskGroupV2.id.in_(used_task_group_ids))
+    )
+    used_task_groups = res.scalars().all()
+    for used_task_group in used_task_groups:
+        used_task_group.timestamp_last_used = job.start_timestamp
+        db.add(used_task_group)
+    await db.commit()
+
     # Define server-side job directory
-    timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+    timestamp_string = job.start_timestamp.strftime("%Y%m%d_%H%M%S")
     WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
         f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
         f"_{timestamp_string}"

fractal_server/app/routes/api/v2/task_collection.py

@@ -17,24 +17,27 @@ from .....logger import set_logger
 from .....syringe import Inject
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models.v2 import CollectionStateV2
 from ....models.v2 import TaskGroupV2
-from ....schemas.v2 import CollectionStateReadV2
-from ....schemas.v2 import CollectionStatusV2
 from ....schemas.v2 import TaskCollectPipV2
-from ....schemas.v2 import TaskGroupCreateV2
+from ....schemas.v2 import TaskGroupActivityStatusV2
+from ....schemas.v2 import TaskGroupActivityV2Read
+from ....schemas.v2 import TaskGroupCreateV2Strict
 from ...aux.validate_user_settings import validate_user_settings
-from ._aux_functions_task_collection import get_package_version_from_pypi
+from ._aux_functions_task_lifecycle import get_package_version_from_pypi
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
 from ._aux_functions_tasks import _verify_non_duplication_user_constraint
 from fractal_server.app.models import UserOAuth
-from fractal_server.app.routes.auth import current_active_user
+from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.schemas.v2 import (
+    TaskGroupActivityActionV2,
+)
 from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
-from fractal_server.tasks.v2.collection_local import (
-    collect_package_local,
+from fractal_server.tasks.v2.local.collect import (
+    collect_local,
 )
+from fractal_server.tasks.v2.ssh import collect_ssh
 from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
 from fractal_server.tasks.v2.utils_package_names import normalize_package_name
 from fractal_server.tasks.v2.utils_python_interpreter import (
@@ -48,7 +51,7 @@ logger = set_logger(__name__)
 
 @router.post(
     "/collect/pip/",
-    response_model=CollectionStateReadV2,
+    response_model=TaskGroupActivityV2Read,
 )
 async def collect_tasks_pip(
     task_collect: TaskCollectPipV2,
@@ -59,12 +62,9 @@ async def collect_tasks_pip(
     user_group_id: Optional[int] = None,
     user: UserOAuth = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
-) -> CollectionStateReadV2:
+) -> TaskGroupActivityV2Read:
     """
-    Task collection endpoint
-
-    Trigger the creation of a dedicated virtual environment, the installation
-    of a package and the collection of tasks as advertised in the manifest.
+    Task-collection endpoint
     """
 
     # Get settings
@@ -164,7 +164,7 @@ async def collect_tasks_pip(
 
     # Validate TaskGroupV2 attributes
     try:
-        TaskGroupCreateV2(**task_group_attrs)
+        TaskGroupCreateV2Strict(**task_group_attrs)
     except ValidationError as e:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -227,32 +227,24 @@ async def collect_tasks_pip(
     db.expunge(task_group)
 
     # All checks are OK, proceed with task collection
-    collection_state_data = dict(
-        status=CollectionStatusV2.PENDING,
-        package=task_group.pkg_name,
+    task_group_activity = TaskGroupActivityV2(
+        user_id=task_group.user_id,
+        taskgroupv2_id=task_group.id,
+        status=TaskGroupActivityStatusV2.PENDING,
+        action=TaskGroupActivityActionV2.COLLECT,
+        pkg_name=task_group.pkg_name,
         version=task_group.version,
-        path=task_group.path,
-        venv_path=task_group.venv_path,
     )
-    state = CollectionStateV2(
-        data=collection_state_data, taskgroupv2_id=task_group.id
-    )
-    db.add(state)
+    db.add(task_group_activity)
     await db.commit()
-    await db.refresh(state)
-
+    await db.refresh(task_group_activity)
     logger = set_logger(logger_name="collect_tasks_pip")
 
     # END of SSH/non-SSH common part
 
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         # SSH task collection
-
-        from fractal_server.tasks.v2.collection_ssh import (
-            collect_package_ssh,
-        )
-
-        # User appropriate FractalSSH object
+        # Use appropriate FractalSSH object
         ssh_credentials = dict(
             user=user_settings.ssh_username,
             host=user_settings.ssh_host,
@@ -262,9 +254,9 @@ async def collect_tasks_pip(
         fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
 
         background_tasks.add_task(
-            collect_package_ssh,
-            state_id=state.id,
-            task_group=task_group,
+            collect_ssh,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
             fractal_ssh=fractal_ssh,
             tasks_base_dir=user_settings.ssh_tasks_dir,
         )
@@ -272,38 +264,14 @@ async def collect_tasks_pip(
     else:
         # Local task collection
         background_tasks.add_task(
-            collect_package_local,
-            state_id=state.id,
-            task_group=task_group,
+            collect_local,
+            task_group_id=task_group.id,
+            task_group_activity_id=task_group_activity.id,
         )
     logger.debug(
         "Task-collection endpoint: start background collection "
-        "and return state"
+        "and return task_group_activity"
     )
     reset_logger_handlers(logger)
-    info = (
-        "Collecting tasks in the background. "
-        f"GET /task/collect/{state.id}/ to query collection status"
-    )
-    state.data["info"] = info
-    response.status_code = status.HTTP_201_CREATED
-
-    return state
-
-
-@router.get("/collect/{state_id}/", response_model=CollectionStateReadV2)
-async def check_collection_status(
-    state_id: int,
-    user: UserOAuth = Depends(current_active_user),
-    db: AsyncSession = Depends(get_async_db),
-) -> CollectionStateReadV2:  # State[TaskCollectStatus]
-    """
-    Check status of background task collection
-    """
-    state = await db.get(CollectionStateV2, state_id)
-    if state is None:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=f"No task collection info with id={state_id}",
-        )
-    return state
+    response.status_code = status.HTTP_202_ACCEPTED
+    return task_group_activity

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -27,12 +27,12 @@ from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
 from fractal_server.syringe import Inject
-from fractal_server.tasks.v2.database_operations import (
-    create_db_tasks_and_update_task_group,
-)
 from fractal_server.tasks.v2.utils_background import (
     _prepare_tasks_metadata,
 )
+from fractal_server.tasks.v2.utils_database import (
+    create_db_tasks_and_update_task_group,
+)
 
 router = APIRouter()
 

fractal_server/app/routes/api/v2/task_group.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+from typing import Optional
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -13,13 +16,17 @@ from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
-from fractal_server.app.models.v2 import CollectionStateV2
+from fractal_server.app.models.v2 import TaskGroupActivityV2
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
+from fractal_server.app.routes.aux import _raise_if_naive_datetime
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityV2Read
 from fractal_server.app.schemas.v2 import TaskGroupReadV2
 from fractal_server.app.schemas.v2 import TaskGroupUpdateV2
 from fractal_server.logger import set_logger
@@ -29,6 +36,72 @@ router = APIRouter()
 logger = set_logger(__name__)
 
 
+@router.get("/activity/", response_model=list[TaskGroupActivityV2Read])
+async def get_task_group_activity_list(
+    task_group_activity_id: Optional[int] = None,
+    taskgroupv2_id: Optional[int] = None,
+    pkg_name: Optional[str] = None,
+    status: Optional[TaskGroupActivityStatusV2] = None,
+    action: Optional[TaskGroupActivityActionV2] = None,
+    timestamp_started_min: Optional[datetime] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskGroupActivityV2Read]:
+
+    _raise_if_naive_datetime(timestamp_started_min)
+
+    stm = select(TaskGroupActivityV2).where(
+        TaskGroupActivityV2.user_id == user.id
+    )
+    if task_group_activity_id is not None:
+        stm = stm.where(TaskGroupActivityV2.id == task_group_activity_id)
+    if taskgroupv2_id is not None:
+        stm = stm.where(TaskGroupActivityV2.taskgroupv2_id == taskgroupv2_id)
+    if pkg_name is not None:
+        stm = stm.where(TaskGroupActivityV2.pkg_name.icontains(pkg_name))
+    if status is not None:
+        stm = stm.where(TaskGroupActivityV2.status == status)
+    if action is not None:
+        stm = stm.where(TaskGroupActivityV2.action == action)
+    if timestamp_started_min is not None:
+        stm = stm.where(
+            TaskGroupActivityV2.timestamp_started >= timestamp_started_min
+        )
+
+    res = await db.execute(stm)
+    activities = res.scalars().all()
+    return activities
+
+
+@router.get(
+    "/activity/{task_group_activity_id}/",
+    response_model=TaskGroupActivityV2Read,
+)
+async def get_task_group_activity(
+    task_group_activity_id: int,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskGroupActivityV2Read:
+
+    activity = await db.get(TaskGroupActivityV2, task_group_activity_id)
+
+    if activity is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"TaskGroupActivityV2 {task_group_activity_id} not found",
+        )
+    if activity.user_id != user.id:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=(
+                "You are not the owner of TaskGroupActivityV2 "
+                f"{task_group_activity_id}",
+            ),
+        )
+
+    return activity
+
+
 @router.get("/", response_model=list[TaskGroupReadV2])
 async def get_task_group_list(
     user: UserOAuth = Depends(current_active_user),
@@ -40,7 +113,6 @@ async def get_task_group_list(
     """
     Get all accessible TaskGroups
     """
-
     if only_owner:
         condition = TaskGroupV2.user_id == user.id
     else:
@@ -112,22 +184,22 @@ async def delete_task_group(
             detail=f"TaskV2 {workflow_tasks[0].task_id} is still in use",
         )
 
-    # Cascade operations: set foreign-keys to null for CollectionStateV2 which
-    # are in relationship with the current TaskGroupV2
-    logger.debug("Start of cascade operations on CollectionStateV2.")
-    stm = select(CollectionStateV2).where(
-        CollectionStateV2.taskgroupv2_id == task_group_id
+    # Cascade operations: set foreign-keys to null for TaskGroupActivityV2
+    # which are in relationship with the current TaskGroupV2
+    logger.debug("Start of cascade operations on TaskGroupActivityV2.")
+    stm = select(TaskGroupActivityV2).where(
+        TaskGroupActivityV2.taskgroupv2_id == task_group_id
     )
     res = await db.execute(stm)
-    collection_states = res.scalars().all()
-    for collection_state in collection_states:
+    task_group_activity_list = res.scalars().all()
+    for task_group_activity in task_group_activity_list:
         logger.debug(
-            f"Setting CollectionStateV2[{collection_state.id}].taskgroupv2_id "
-            "to None."
+            f"Setting TaskGroupActivityV2[{task_group_activity.id}]"
+            ".taskgroupv2_id to None."
         )
-        collection_state.taskgroupv2_id = None
-        db.add(collection_state)
-    logger.debug("End of cascade operations on CollectionStateV2.")
+        task_group_activity.taskgroupv2_id = None
+        db.add(task_group_activity)
+    logger.debug("End of cascade operations on TaskGroupActivityV2.")
 
     await db.delete(task_group)
     await db.commit()