fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/job.py

@@ -0,0 +1,200 @@
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Response
+from fastapi import status
+from fastapi.responses import StreamingResponse
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import JobV2
+from ....models.v2 import ProjectV2
+from ....runner.filenames import WORKFLOW_LOG_FILENAME  # FIXME
+from ....schemas.v2 import JobReadV2
+from ....schemas.v2 import JobStatusTypeV2
+from ....security import current_active_user
+from ....security import User
+from ...aux._job import _write_shutdown_file
+from ...aux._job import _zip_folder_to_byte_stream
+from ...aux._runner import _check_backend_is_slurm
+from ._aux_functions import _get_job_check_owner
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_workflow_check_owner
+
+router = APIRouter()
+
+
+@router.get("/job/", response_model=list[JobReadV2])
+async def get_user_jobs(
+    user: User = Depends(current_active_user),
+    log: bool = True,
+    db: AsyncSession = Depends(get_async_db),
+) -> list[JobReadV2]:
+    """
+    Returns all the jobs of the current user
+    """
+    stm = (
+        select(JobV2)
+        .join(ProjectV2)
+        .where(ProjectV2.user_list.any(User.id == user.id))
+    )
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+    if not log:
+        for job in job_list:
+            setattr(job, "log", None)
+
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/job/",
+    response_model=list[JobReadV2],
+)
+async def get_workflow_jobs(
+    project_id: int,
+    workflow_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[list[JobReadV2]]:
+    """
+    Returns all the jobs related to a specific workflow
+    """
+    await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+    stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/",
+    response_model=JobReadV2,
+)
+async def read_job(
+    project_id: int,
+    job_id: int,
+    show_tmp_logs: bool = False,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[JobReadV2]:
+    """
+    Return info on an existing job
+    """
+
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+    await db.close()
+
+    if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
+        try:
+            with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
+                job.log = f.read()
+        except FileNotFoundError:
+            pass
+
+    return job
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/download/",
+    response_class=StreamingResponse,
+)
+async def download_job_logs(
+    project_id: int,
+    job_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> StreamingResponse:
+    """
+    Download job folder
+    """
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+
+    # Create and return byte stream for zipped log folder
+    PREFIX_ZIP = Path(job.working_dir).name
+    zip_filename = f"{PREFIX_ZIP}_archive.zip"
+    byte_stream = _zip_folder_to_byte_stream(
+        folder=job.working_dir, zip_filename=zip_filename
+    )
+    return StreamingResponse(
+        iter([byte_stream.getvalue()]),
+        media_type="application/x-zip-compressed",
+        headers={"Content-Disposition": f"attachment;filename={zip_filename}"},
+    )
+
+
+@router.get(
+    "/project/{project_id}/job/",
+    response_model=list[JobReadV2],
+)
+async def get_job_list(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    log: bool = True,
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[list[JobReadV2]]:
+    """
+    Get job list for given project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    stm = select(JobV2).where(JobV2.project_id == project.id)
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+    if not log:
+        for job in job_list:
+            setattr(job, "log", None)
+
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/stop/",
+    status_code=202,
+)
+async def stop_job(
+    project_id: int,
+    job_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Stop execution of a workflow job (only available for slurm backend)
+    """
+
+    # This endpoint is only implemented for SLURM backend
+    _check_backend_is_slurm()
+
+    # Get job from DB
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+
+    _write_shutdown_file(job=job)
+
+    return Response(status_code=status.HTTP_202_ACCEPTED)

fractal_server/app/routes/api/v2/project.py

@@ -0,0 +1,205 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlalchemy.exc import IntegrityError
+from sqlmodel import select
+
+from .....logger import close_logger
+from .....logger import set_logger
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import DatasetV2
+from ....models.v2 import JobV2
+from ....models.v2 import LinkUserProjectV2
+from ....models.v2 import ProjectV2
+from ....models.v2 import WorkflowV2
+from ....schemas.v2 import ProjectCreateV2
+from ....schemas.v2 import ProjectReadV2
+from ....schemas.v2 import ProjectUpdateV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _check_project_exists
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+
+router = APIRouter()
+
+
+@router.get("/project/", response_model=list[ProjectReadV2])
+async def get_list_project(
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[ProjectV2]:
+    """
+    Return list of projects user is member of
+    """
+    stm = (
+        select(ProjectV2)
+        .join(LinkUserProjectV2)
+        .where(LinkUserProjectV2.user_id == user.id)
+    )
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+    return project_list
+
+
+@router.post("/project/", response_model=ProjectReadV2, status_code=201)
+async def create_project(
+    project: ProjectCreateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Create new poject
+    """
+
+    # Check that there is no project with the same user and name
+    await _check_project_exists(
+        project_name=project.name, user_id=user.id, db=db
+    )
+
+    db_project = ProjectV2(**project.dict())
+    db_project.user_list.append(user)
+    try:
+        db.add(db_project)
+        await db.commit()
+        await db.refresh(db_project)
+        await db.close()
+    except IntegrityError as e:
+        await db.rollback()
+        logger = set_logger("create_project")
+        logger.error(str(e))
+        close_logger(logger)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+    return db_project
+
+
+@router.get("/project/{project_id}/", response_model=ProjectReadV2)
+async def read_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Return info on an existing project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+    await db.close()
+    return project
+
+
+@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
+async def update_project(
+    project_id: int,
+    project_update: ProjectUpdateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Check that there is no project with the same user and name
+    if project_update.name is not None:
+        await _check_project_exists(
+            project_name=project_update.name, user_id=user.id, db=db
+        )
+
+    for key, value in project_update.dict(exclude_unset=True).items():
+        setattr(project, key, value)
+
+    await db.commit()
+    await db.refresh(project)
+    await db.close()
+    return project
+
+
+@router.delete("/project/{project_id}/", status_code=204)
+async def delete_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current project.
+    stm = _get_submitted_jobs_statement().where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    if jobs:
+        string_ids = str([job.id for job in jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot delete project {project.id} because it "
+                f"is linked to active job(s) {string_ids}."
+            ),
+        )
+
+    # Cascade operations
+
+    # Workflows
+    stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
+    res = await db.execute(stm)
+    workflows = res.scalars().all()
+    for wf in workflows:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current workflow
+        stm = select(JobV2).where(JobV2.workflow_id == wf.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.workflow_id = None
+            await db.merge(job)
+        # Delete workflow
+        await db.delete(wf)
+    await db.commit()
+
+    # Dataset
+    stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
+    res = await db.execute(stm)
+    datasets = res.scalars().all()
+    for ds in datasets:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current dataset
+        stm = select(JobV2).where(JobV2.dataset_id == ds.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.dataset_id = None
+            await db.merge(job)
+        # Delete dataset
+        await db.delete(ds)
+    await db.commit()
+
+    # Job
+    stm = select(JobV2).where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    for job in jobs:
+        job.project_id = None
+        await db.merge(job)
+
+    await db.commit()
+
+    await db.delete(project)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v2/task.py

@@ -0,0 +1,222 @@
+from copy import deepcopy  # noqa
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import select
+
+from .....logger import set_logger
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v1 import Task as TaskV1
+from ....models.v2 import TaskV2
+from ....models.v2 import WorkflowTaskV2
+from ....schemas.v2 import TaskCreateV2
+from ....schemas.v2 import TaskReadV2
+from ....schemas.v2 import TaskUpdateV2
+from ....security import current_active_user
+from ....security import current_active_verified_user
+from ....security import User
+from ._aux_functions import _get_task_check_owner
+
+router = APIRouter()
+
+logger = set_logger(__name__)
+
+
+@router.get("/", response_model=list[TaskReadV2])
+async def get_list_task(
+    args_schema_parallel: bool = True,
+    args_schema_non_parallel: bool = True,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskReadV2]:
+    """
+    Get list of available tasks
+    """
+    stm = select(TaskV2)
+    res = await db.execute(stm)
+    task_list = res.scalars().all()
+    await db.close()
+    if args_schema_parallel is False:
+        for task in task_list:
+            setattr(task, "args_schema_parallel", None)
+    if args_schema_non_parallel is False:
+        for task in task_list:
+            setattr(task, "args_schema_non_parallel", None)
+
+    return task_list
+
+
+@router.get("/{task_id}/", response_model=TaskReadV2)
+async def get_task(
+    task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskReadV2:
+    """
+    Get info on a specific task
+    """
+    task = await db.get(TaskV2, task_id)
+    await db.close()
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail="TaskV2 not found"
+        )
+    return task
+
+
+@router.patch("/{task_id}/", response_model=TaskReadV2)
+async def patch_task(
+    task_id: int,
+    task_update: TaskUpdateV2,
+    user: User = Depends(current_active_verified_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[TaskReadV2]:
+    """
+    Edit a specific task (restricted to superusers and task owner)
+    """
+
+    # Retrieve task from database
+    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
+    update = task_update.dict(exclude_unset=True)
+
+    # Forbid changes that set a previously unset command
+    if db_task.type == "non_parallel" and "command_parallel" in update:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set an unset `command_parallel`.",
+        )
+    if db_task.type == "parallel" and "command_non_parallel" in update:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set an unset `command_non_parallel`.",
+        )
+
+    for key, value in update.items():
+        setattr(db_task, key, value)
+
+    await db.commit()
+    await db.refresh(db_task)
+    await db.close()
+    return db_task
+
+
+@router.post(
+    "/", response_model=TaskReadV2, status_code=status.HTTP_201_CREATED
+)
+async def create_task(
+    task: TaskCreateV2,
+    user: User = Depends(current_active_verified_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[TaskReadV2]:
+    """
+    Create a new task
+    """
+
+    if task.command_non_parallel is None:
+        task_type = "parallel"
+    elif task.command_parallel is None:
+        task_type = "non_parallel"
+    else:
+        task_type = "compound"
+
+    if task_type == "parallel" and (
+        task.args_schema_non_parallel is not None
+        or task.meta_non_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot set `TaskV2.args_schema_non_parallel` or "
+                "`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
+            ),
+        )
+    elif task_type == "non_parallel" and (
+        task.args_schema_parallel is not None or task.meta_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot set `TaskV2.args_schema_parallel` or "
+                "`TaskV2.args_schema_parallel` if TaskV2 is non_parallel"
+            ),
+        )
+
+    # Set task.owner attribute
+    if user.username:
+        owner = user.username
+    elif user.slurm_user:
+        owner = user.slurm_user
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot add a new task because current user does not "
+                "have `username` or `slurm_user` attributes."
+            ),
+        )
+
+    # Prepend owner to task.source
+    task.source = f"{owner}:{task.source}"
+
+    # Verify that source is not already in use (note: this check is only useful
+    # to provide a user-friendly error message, but `task.source` uniqueness is
+    # already guaranteed by a constraint in the table definition).
+    stm = select(TaskV2).where(TaskV2.source == task.source)
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Source '{task.source}' already used by some TaskV2",
+        )
+    stm = select(TaskV1).where(TaskV1.source == task.source)
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Source '{task.source}' already used by some TaskV1",
+        )
+    # Add task
+    db_task = TaskV2(**task.dict(), owner=owner, type=task_type)
+    db.add(db_task)
+    await db.commit()
+    await db.refresh(db_task)
+    await db.close()
+    return db_task
+
+
+@router.delete("/{task_id}/", status_code=204)
+async def delete_task(
+    task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete a task
+    """
+
+    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
+
+    # Check that the TaskV2 is not in relationship with some WorkflowTaskV2
+    stm = select(WorkflowTaskV2).filter(WorkflowTaskV2.task_id == task_id)
+    res = await db.execute(stm)
+    workflowtask_list = res.scalars().all()
+    if workflowtask_list:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot remove TaskV2 {task_id} because it is currently "
+                "imported in WorkflowsV2 "
+                f"{[x.workflow_id for x in workflowtask_list]}. "
+                "If you want to remove this task, then you should first remove"
+                " the workflows.",
+            ),
+        )
+
+    await db.delete(db_task)
+    await db.commit()
+    return Response(status_code=status.HTTP_204_NO_CONTENT)