fractal-server 2.0.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.0.6"
+__VERSION__ = "2.2.0"

fractal_server/app/db/__init__.py

@@ -63,7 +63,7 @@ class DB:
             }
 
         cls._engine_async = create_async_engine(
-            settings.DATABASE_URL,
+            settings.DATABASE_ASYNC_URL,
             echo=settings.DB_ECHO,
             future=True,
             **engine_kwargs_async,

fractal_server/app/routes/admin/v1.py

@@ -35,7 +35,7 @@ from ...schemas.v1 import WorkflowReadV1
 from ...security import current_active_superuser
 from ..aux._job import _write_shutdown_file
 from ..aux._job import _zip_folder_to_byte_stream
-from ..aux._runner import _check_backend_is_slurm
+from ..aux._runner import _is_shutdown_available
 
 router_admin_v1 = APIRouter()
 
@@ -349,11 +349,9 @@ async def stop_job(
 ) -> Response:
     """
     Stop execution of a workflow job.
-
-    Only available for slurm backend.
     """
 
-    _check_backend_is_slurm()
+    _is_shutdown_available()
 
     job = await db.get(ApplyWorkflow, job_id)
     if job is None:

fractal_server/app/routes/admin/v2.py

@@ -38,7 +38,7 @@ from ...schemas.v2 import ProjectReadV2
 from ...security import current_active_superuser
 from ..aux._job import _write_shutdown_file
 from ..aux._job import _zip_folder_to_byte_stream
-from ..aux._runner import _check_backend_is_slurm
+from ..aux._runner import _is_shutdown_available
 
 router_admin_v2 = APIRouter()
 
@@ -236,11 +236,9 @@ async def stop_job(
 ) -> Response:
     """
     Stop execution of a workflow job.
-
-    Only available for slurm backend.
     """
 
-    _check_backend_is_slurm()
+    _is_shutdown_available()
 
     job = await db.get(JobV2, job_id)
     if job is None:

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -436,3 +436,27 @@ async def _workflow_insert_task(
     await db.refresh(wf_task)
 
     return wf_task
+
+
+async def clean_app_job_list_v1(
+    db: AsyncSession, jobs_list: list[int]
+) -> list[int]:
+    """
+    Remove from a job list all jobs with status different from submitted.
+
+    Args:
+        db: Async database session
+        jobs_list: List of job IDs currently associated to the app.
+
+    Return:
+        List of IDs for submitted jobs.
+    """
+    stmt = select(ApplyWorkflow).where(ApplyWorkflow.id.in_(jobs_list))
+    result = await db.execute(stmt)
+    db_jobs_list = result.scalars().all()
+    submitted_job_ids = [
+        job.id
+        for job in db_jobs_list
+        if job.status == JobStatusTypeV1.SUBMITTED
+    ]
+    return submitted_job_ids

fractal_server/app/routes/api/v1/job.py

@@ -19,7 +19,7 @@ from ....security import current_active_user
 from ....security import User
 from ...aux._job import _write_shutdown_file
 from ...aux._job import _zip_folder_to_byte_stream
-from ...aux._runner import _check_backend_is_slurm
+from ...aux._runner import _is_shutdown_available
 from ._aux_functions import _get_job_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_workflow_check_owner
@@ -177,11 +177,10 @@ async def stop_job(
     db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
-    Stop execution of a workflow job (only available for slurm backend)
+    Stop execution of a workflow job.
     """
 
-    # This endpoint is only implemented for SLURM backend
-    _check_backend_is_slurm()
+    _is_shutdown_available()
 
     # Get job from DB
     output = await _get_job_check_owner(

fractal_server/app/routes/api/v1/project.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -7,13 +8,12 @@ from fastapi import APIRouter
 from fastapi import BackgroundTasks
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Request
 from fastapi import Response
 from fastapi import status
-from sqlalchemy.exc import IntegrityError
 from sqlmodel import select
 
 from .....config import get_settings
-from .....logger import close_logger
 from .....logger import set_logger
 from .....syringe import Inject
 from ....db import AsyncSession
@@ -42,8 +42,10 @@ from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import clean_app_job_list_v1
 
 router = APIRouter()
+logger = set_logger(__name__)
 
 
 def _encode_as_utc(dt: datetime):
@@ -86,20 +88,11 @@ async def create_project(
 
     db_project = Project(**project.dict())
     db_project.user_list.append(user)
-    try:
-        db.add(db_project)
-        await db.commit()
-        await db.refresh(db_project)
-        await db.close()
-    except IntegrityError as e:
-        await db.rollback()
-        logger = set_logger("create_project")
-        logger.error(str(e))
-        close_logger(logger)
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=str(e),
-        )
+
+    db.add(db_project)
+    await db.commit()
+    await db.refresh(db_project)
+    await db.close()
 
     return db_project
 
@@ -252,10 +245,23 @@ async def apply_workflow(
     background_tasks: BackgroundTasks,
     input_dataset_id: int,
     output_dataset_id: int,
+    request: Request,
     user: User = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ApplyWorkflowReadV1]:
 
+    # Remove non-submitted V1 jobs from the app state when the list grows
+    # beyond a threshold
+    settings = Inject(get_settings)
+    if (
+        len(request.app.state.jobsV1)
+        > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH
+    ):
+        new_jobs_list = await clean_app_job_list_v1(
+            db, request.app.state.jobsV1
+        )
+        request.app.state.jobsV1 = new_jobs_list
+
     output = await _get_dataset_check_owner(
         project_id=project_id,
         dataset_id=input_dataset_id,
@@ -312,7 +318,6 @@ async def apply_workflow(
         )
 
     # If backend is SLURM, check that the user has required attributes
-    settings = Inject(get_settings)
     backend = settings.FRACTAL_RUNNER_BACKEND
     if backend == "slurm":
         if not user.slurm_user:
@@ -474,7 +479,12 @@ async def apply_workflow(
         slurm_user=user.slurm_user,
         user_cache_dir=user.cache_dir,
     )
-
+    request.app.state.jobsV1.append(job.id)
+    logger.info(
+        f"Current worker's pid is {os.getpid()}. "
+        f"Current status of worker job's list "
+        f"{request.app.state.jobsV1}"
+    )
     await db.close()
 
     return job

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -95,8 +95,11 @@ async def _get_workflow_check_owner(
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user_id, db=db
     )
+
     # Get workflow
-    workflow = await db.get(WorkflowV2, workflow_id)
+    # (See issue 1087 for 'populate_existing=True')
+    workflow = await db.get(WorkflowV2, workflow_id, populate_existing=True)
+
     if not workflow:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Workflow not found"
@@ -107,10 +110,6 @@ async def _get_workflow_check_owner(
             detail=(f"Invalid {project_id=} for {workflow_id=}."),
         )
 
-    # Refresh so that workflow.project relationship is loaded (see discussion
-    # in issue #1063)
-    await db.refresh(workflow)
-
     return workflow
 
 
@@ -255,13 +254,15 @@ async def _get_dataset_check_owner(
         HTTPException(status_code=422_UNPROCESSABLE_ENTITY):
             If the dataset is not associated to the project
     """
-
     # Access control for project
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user_id, db=db
     )
+
     # Get dataset
-    dataset = await db.get(DatasetV2, dataset_id)
+    # (See issue 1087 for 'populate_existing=True')
+    dataset = await db.get(DatasetV2, dataset_id, populate_existing=True)
+
     if not dataset:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND, detail="Dataset not found"
@@ -272,10 +273,6 @@ async def _get_dataset_check_owner(
             detail=f"Invalid {project_id=} for {dataset_id=}",
         )
 
-    # Refresh so that dataset.project relationship is loaded (see discussion
-    # in issue #1063)
-    await db.refresh(dataset)
-
     return dict(dataset=dataset, project=project)
 
 
@@ -497,6 +494,32 @@ async def _workflow_insert_task(
     db_workflow.task_list.reorder()  # type: ignore
     flag_modified(db_workflow, "task_list")
     await db.commit()
-    await db.refresh(wf_task)
+
+    # See issue 1087 for 'populate_existing=True'
+    wf_task = await db.get(WorkflowTaskV2, wf_task.id, populate_existing=True)
 
     return wf_task
+
+
+async def clean_app_job_list_v2(
+    db: AsyncSession, jobs_list: list[int]
+) -> list[int]:
+    """
+    Remove from a job list all jobs with status different from submitted.
+
+    Args:
+        db: Async database session
+        jobs_list: List of job IDs currently associated to the app.
+
+    Return:
+        List of IDs for submitted jobs.
+    """
+    stmt = select(JobV2).where(JobV2.id.in_(jobs_list))
+    result = await db.execute(stmt)
+    db_jobs_list = result.scalars().all()
+    submitted_job_ids = [
+        job.id
+        for job in db_jobs_list
+        if job.status == JobStatusTypeV2.SUBMITTED
+    ]
+    return submitted_job_ids

fractal_server/app/routes/api/v2/job.py

@@ -19,7 +19,7 @@ from ....security import current_active_user
 from ....security import User
 from ...aux._job import _write_shutdown_file
 from ...aux._job import _zip_folder_to_byte_stream
-from ...aux._runner import _check_backend_is_slurm
+from ...aux._runner import _is_shutdown_available
 from ._aux_functions import _get_job_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_workflow_check_owner
@@ -180,11 +180,10 @@ async def stop_job(
     db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
-    Stop execution of a workflow job (only available for slurm backend)
+    Stop execution of a workflow job.
     """
 
-    # This endpoint is only implemented for SLURM backend
-    _check_backend_is_slurm()
+    _is_shutdown_available()
 
     # Get job from DB
     output = await _get_job_check_owner(

fractal_server/app/routes/api/v2/project.py

@@ -7,6 +7,8 @@ from fastapi import Response
 from fastapi import status
 from sqlmodel import select
 
+from .....logger import reset_logger_handlers
+from .....logger import set_logger
 from ....db import AsyncSession
 from ....db import get_async_db
 from ....models.v2 import DatasetV2
@@ -122,9 +124,11 @@ async def delete_project(
     """
     Delete project
     """
+
     project = await _get_project_check_owner(
         project_id=project_id, user_id=user.id, db=db
     )
+    logger = set_logger(__name__)
 
     # Fail if there exist jobs that are submitted and in relation with the
     # current project.
@@ -147,6 +151,7 @@ async def delete_project(
     stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
     res = await db.execute(stm)
     workflows = res.scalars().all()
+    logger.info("Start of cascade operations on Workflows.")
     for wf in workflows:
         # Cascade operations: set foreign-keys to null for jobs which are in
         # relationship with the current workflow
@@ -154,14 +159,18 @@ async def delete_project(
         res = await db.execute(stm)
         jobs = res.scalars().all()
         for job in jobs:
+            logger.info(f"Setting Job[{job.id}].workflow_id to None.")
             job.workflow_id = None
         # Delete workflow
+        logger.info(f"Adding Workflow[{wf.id}] to deletion.")
         await db.delete(wf)
+    logger.info("End of cascade operations on Workflows.")
 
     # Dataset
     stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
     res = await db.execute(stm)
     datasets = res.scalars().all()
+    logger.info("Start of cascade operations on Datasets.")
     for ds in datasets:
         # Cascade operations: set foreign-keys to null for jobs which are in
         # relationship with the current dataset
@@ -169,18 +178,30 @@ async def delete_project(
         res = await db.execute(stm)
         jobs = res.scalars().all()
         for job in jobs:
+            logger.info(f"Setting Job[{job.id}].dataset_id to None.")
             job.dataset_id = None
         # Delete dataset
+        logger.info(f"Adding Dataset[{ds.id}] to deletion.")
         await db.delete(ds)
+    logger.info("End of cascade operations on Datasets.")
 
     # Job
+    logger.info("Start of cascade operations on Jobs.")
     stm = select(JobV2).where(JobV2.project_id == project_id)
     res = await db.execute(stm)
     jobs = res.scalars().all()
     for job in jobs:
+        logger.info(f"Setting Job[{job.id}].project_id to None.")
         job.project_id = None
+    logger.info("End of cascade operations on Jobs.")
 
+    logger.info(f"Adding Project[{project.id}] to deletion.")
     await db.delete(project)
+
+    logger.info("Committing changes to db...")
     await db.commit()
 
+    logger.info("Everything  has been deleted correctly.")
+    reset_logger_handlers(logger)
+
     return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v2/submit.py

@@ -1,3 +1,4 @@
+import os
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -8,10 +9,12 @@ from fastapi import APIRouter
 from fastapi import BackgroundTasks
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Request
 from fastapi import status
 from sqlmodel import select
 
 from .....config import get_settings
+from .....logger import set_logger
 from .....syringe import Inject
 from .....utils import get_timestamp
 from ....db import AsyncSession
@@ -28,6 +31,7 @@ from ....security import current_active_verified_user
 from ....security import User
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
+from ._aux_functions import clean_app_job_list_v2
 
 
 def _encode_as_utc(dt: datetime):
@@ -35,6 +39,7 @@ def _encode_as_utc(dt: datetime):
 
 
 router = APIRouter()
+logger = set_logger(__name__)
 
 
 @router.post(
@@ -48,10 +53,23 @@ async def apply_workflow(
     dataset_id: int,
     job_create: JobCreateV2,
     background_tasks: BackgroundTasks,
+    request: Request,
     user: User = Depends(current_active_verified_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> Optional[JobReadV2]:
 
+    # Remove non-submitted V2 jobs from the app state when the list grows
+    # beyond a threshold
+    settings = Inject(get_settings)
+    if (
+        len(request.app.state.jobsV2)
+        > settings.FRACTAL_API_MAX_JOB_LIST_LENGTH
+    ):
+        new_jobs_list = await clean_app_job_list_v2(
+            db, request.app.state.jobsV2
+        )
+        request.app.state.jobsV2 = new_jobs_list
+
     output = await _get_dataset_check_owner(
         project_id=project_id,
         dataset_id=dataset_id,
@@ -92,7 +110,6 @@ async def apply_workflow(
         )
 
     # If backend is SLURM, check that the user has required attributes
-    settings = Inject(get_settings)
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     if FRACTAL_RUNNER_BACKEND == "slurm":
         if not user.slurm_user:
@@ -195,25 +212,24 @@ async def apply_workflow(
 
     # Define server-side job directory
     timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
-    WORKFLOW_DIR = (
-        settings.FRACTAL_RUNNER_WORKING_BASE_DIR
-        / (
-            f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
-            f"_{timestamp_string}"
-        )
-    ).resolve()
+    WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
+        f"proj_v2_{project_id:07d}_wf_{workflow_id:07d}_job_{job.id:07d}"
+        f"_{timestamp_string}"
+    )
 
     # Define user-side job directory
     if FRACTAL_RUNNER_BACKEND == "local":
-        WORKFLOW_DIR_USER = WORKFLOW_DIR
+        WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+    elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+        WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
     elif FRACTAL_RUNNER_BACKEND == "slurm":
-        WORKFLOW_DIR_USER = (
-            Path(user.cache_dir) / f"{WORKFLOW_DIR.name}"
-        ).resolve()
+        WORKFLOW_DIR_REMOTE = (
+            Path(user.cache_dir) / f"{WORKFLOW_DIR_LOCAL.name}"
+        )
 
     # Update job folders in the db
-    job.working_dir = WORKFLOW_DIR.as_posix()
-    job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+    job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
+    job.working_dir_user = WORKFLOW_DIR_REMOTE.as_posix()
     await db.merge(job)
     await db.commit()
 
@@ -226,6 +242,11 @@ async def apply_workflow(
         slurm_user=user.slurm_user,
         user_cache_dir=user.cache_dir,
     )
-
+    request.app.state.jobsV2.append(job.id)
+    logger.info(
+        f"Current worker's pid is {os.getpid()}. "
+        f"Current status of worker job's list "
+        f"{request.app.state.jobsV2}"
+    )
     await db.close()
     return job

fractal_server/app/routes/aux/_job.py

@@ -1,13 +1,15 @@
 from io import BytesIO
 from pathlib import Path
+from typing import Union
 from zipfile import ZIP_DEFLATED
 from zipfile import ZipFile
 
 from ...models.v1 import ApplyWorkflow
+from ...models.v2 import JobV2
 from ...runner.filenames import SHUTDOWN_FILENAME
 
 
-def _write_shutdown_file(*, job: ApplyWorkflow):
+def _write_shutdown_file(*, job: Union[ApplyWorkflow, JobV2]):
     """
     Write job's shutdown file.
 

fractal_server/app/routes/aux/_runner.py

@@ -5,15 +5,15 @@ from ....config import get_settings
 from ....syringe import Inject
 
 
-def _check_backend_is_slurm():
+def _is_shutdown_available():
     """
     Raises:
         HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY):
-            If FRACTAL_RUNNER_BACKEND is not 'slurm'
+            If FRACTAL_RUNNER_BACKEND is the thread-based 'local' backend.
     """
     settings = Inject(get_settings)
     backend = settings.FRACTAL_RUNNER_BACKEND
-    if backend != "slurm":
+    if backend not in ["slurm", "local_experimental"]:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(