fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/routes/api/v2/job.py

@@ -0,0 +1,200 @@
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import Response
+from fastapi import status
+from fastapi.responses import StreamingResponse
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import JobV2
+from ....models.v2 import ProjectV2
+from ....runner.filenames import WORKFLOW_LOG_FILENAME
+from ....schemas.v2 import JobReadV2
+from ....schemas.v2 import JobStatusTypeV2
+from ....security import current_active_user
+from ....security import User
+from ...aux._job import _write_shutdown_file
+from ...aux._job import _zip_folder_to_byte_stream
+from ...aux._runner import _check_backend_is_slurm
+from ._aux_functions import _get_job_check_owner
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_workflow_check_owner
+
+router = APIRouter()
+
+
+@router.get("/job/", response_model=list[JobReadV2])
+async def get_user_jobs(
+    user: User = Depends(current_active_user),
+    log: bool = True,
+    db: AsyncSession = Depends(get_async_db),
+) -> list[JobReadV2]:
+    """
+    Returns all the jobs of the current user
+    """
+    stm = (
+        select(JobV2)
+        .join(ProjectV2)
+        .where(ProjectV2.user_list.any(User.id == user.id))
+    )
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+    if not log:
+        for job in job_list:
+            setattr(job, "log", None)
+
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/job/",
+    response_model=list[JobReadV2],
+)
+async def get_workflow_jobs(
+    project_id: int,
+    workflow_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[list[JobReadV2]]:
+    """
+    Returns all the jobs related to a specific workflow
+    """
+    await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+    stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/",
+    response_model=JobReadV2,
+)
+async def read_job(
+    project_id: int,
+    job_id: int,
+    show_tmp_logs: bool = False,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[JobReadV2]:
+    """
+    Return info on an existing job
+    """
+
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+    await db.close()
+
+    if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
+        try:
+            with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
+                job.log = f.read()
+        except FileNotFoundError:
+            pass
+
+    return job
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/download/",
+    response_class=StreamingResponse,
+)
+async def download_job_logs(
+    project_id: int,
+    job_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> StreamingResponse:
+    """
+    Download job folder
+    """
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+
+    # Create and return byte stream for zipped log folder
+    PREFIX_ZIP = Path(job.working_dir).name
+    zip_filename = f"{PREFIX_ZIP}_archive.zip"
+    byte_stream = _zip_folder_to_byte_stream(
+        folder=job.working_dir, zip_filename=zip_filename
+    )
+    return StreamingResponse(
+        iter([byte_stream.getvalue()]),
+        media_type="application/x-zip-compressed",
+        headers={"Content-Disposition": f"attachment;filename={zip_filename}"},
+    )
+
+
+@router.get(
+    "/project/{project_id}/job/",
+    response_model=list[JobReadV2],
+)
+async def get_job_list(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    log: bool = True,
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[list[JobReadV2]]:
+    """
+    Get job list for given project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    stm = select(JobV2).where(JobV2.project_id == project.id)
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+    if not log:
+        for job in job_list:
+            setattr(job, "log", None)
+
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/job/{job_id}/stop/",
+    status_code=202,
+)
+async def stop_job(
+    project_id: int,
+    job_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Stop execution of a workflow job (only available for slurm backend)
+    """
+
+    # This endpoint is only implemented for SLURM backend
+    _check_backend_is_slurm()
+
+    # Get job from DB
+    output = await _get_job_check_owner(
+        project_id=project_id,
+        job_id=job_id,
+        user_id=user.id,
+        db=db,
+    )
+    job = output["job"]
+
+    _write_shutdown_file(job=job)
+
+    return Response(status_code=status.HTTP_202_ACCEPTED)

fractal_server/app/routes/api/v2/project.py

@@ -0,0 +1,186 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import select
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import DatasetV2
+from ....models.v2 import JobV2
+from ....models.v2 import LinkUserProjectV2
+from ....models.v2 import ProjectV2
+from ....models.v2 import WorkflowV2
+from ....schemas.v2 import ProjectCreateV2
+from ....schemas.v2 import ProjectReadV2
+from ....schemas.v2 import ProjectUpdateV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _check_project_exists
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+
+router = APIRouter()
+
+
+@router.get("/project/", response_model=list[ProjectReadV2])
+async def get_list_project(
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[ProjectV2]:
+    """
+    Return list of projects user is member of
+    """
+    stm = (
+        select(ProjectV2)
+        .join(LinkUserProjectV2)
+        .where(LinkUserProjectV2.user_id == user.id)
+    )
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+    return project_list
+
+
+@router.post("/project/", response_model=ProjectReadV2, status_code=201)
+async def create_project(
+    project: ProjectCreateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Create new poject
+    """
+
+    # Check that there is no project with the same user and name
+    await _check_project_exists(
+        project_name=project.name, user_id=user.id, db=db
+    )
+
+    db_project = ProjectV2(**project.dict())
+    db_project.user_list.append(user)
+
+    db.add(db_project)
+    await db.commit()
+    await db.refresh(db_project)
+    await db.close()
+
+    return db_project
+
+
+@router.get("/project/{project_id}/", response_model=ProjectReadV2)
+async def read_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Return info on an existing project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+    await db.close()
+    return project
+
+
+@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
+async def update_project(
+    project_id: int,
+    project_update: ProjectUpdateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Check that there is no project with the same user and name
+    if project_update.name is not None:
+        await _check_project_exists(
+            project_name=project_update.name, user_id=user.id, db=db
+        )
+
+    for key, value in project_update.dict(exclude_unset=True).items():
+        setattr(project, key, value)
+
+    await db.commit()
+    await db.refresh(project)
+    await db.close()
+    return project
+
+
+@router.delete("/project/{project_id}/", status_code=204)
+async def delete_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current project.
+    stm = _get_submitted_jobs_statement().where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    if jobs:
+        string_ids = str([job.id for job in jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot delete project {project.id} because it "
+                f"is linked to active job(s) {string_ids}."
+            ),
+        )
+
+    # Cascade operations
+
+    # Workflows
+    stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
+    res = await db.execute(stm)
+    workflows = res.scalars().all()
+    for wf in workflows:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current workflow
+        stm = select(JobV2).where(JobV2.workflow_id == wf.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.workflow_id = None
+        # Delete workflow
+        await db.delete(wf)
+
+    # Dataset
+    stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
+    res = await db.execute(stm)
+    datasets = res.scalars().all()
+    for ds in datasets:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current dataset
+        stm = select(JobV2).where(JobV2.dataset_id == ds.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.dataset_id = None
+        # Delete dataset
+        await db.delete(ds)
+
+    # Job
+    stm = select(JobV2).where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    for job in jobs:
+        job.project_id = None
+
+    await db.delete(project)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v2/status.py

@@ -0,0 +1,150 @@
+import json
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import status
+
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import JobV2
+from ....schemas.v2.dataset import WorkflowTaskStatusTypeV2
+from ....schemas.v2.status import StatusReadV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+from ._aux_functions import _get_workflow_check_owner
+from fractal_server.app.runner.filenames import HISTORY_FILENAME
+
+router = APIRouter()
+
+
+@router.get(
+    "/project/{project_id}/status/",
+    response_model=StatusReadV2,
+)
+async def get_workflowtask_status(
+    project_id: int,
+    dataset_id: int,
+    workflow_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[StatusReadV2]:
+    """
+    Extract the status of all `WorkflowTaskV2` of a given `WorkflowV2` that ran
+    on a given `DatasetV2`.
+
+    *NOTE*: the current endpoint is not guaranteed to provide consistent
+    results if the workflow task list is modified in a non-trivial way
+    (that is, by adding intermediate tasks, removing tasks, or changing their
+    order). See fractal-server GitHub issues: 793, 1083.
+    """
+    # Get the dataset DB entry
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    dataset = output["dataset"]
+
+    # Get the workflow DB entry
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    # Check whether there exists a submitted job associated to this
+    # workflow/dataset pair. If it does exist, it will be used later.
+    # If there are multiple jobs, raise an error.
+    stm = _get_submitted_jobs_statement()
+    stm = stm.where(JobV2.dataset_id == dataset_id)
+    stm = stm.where(JobV2.workflow_id == workflow_id)
+    res = await db.execute(stm)
+    running_jobs = res.scalars().all()
+    if len(running_jobs) == 0:
+        running_job = None
+    elif len(running_jobs) == 1:
+        running_job = running_jobs[0]
+    else:
+        string_ids = str([job.id for job in running_jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot get WorkflowTaskV2 statuses as DatasetV2 {dataset.id}"
+                f" is linked to multiple active jobs: {string_ids}."
+            ),
+        )
+
+    # Initialize empty dictionary for WorkflowTaskV2 status
+    workflow_tasks_status_dict: dict = {}
+
+    # Lowest priority: read status from DB, which corresponds to jobs that are
+    # not running
+    history = dataset.history
+    for history_item in history:
+        wftask_id = history_item["workflowtask"]["id"]
+        wftask_status = history_item["status"]
+        workflow_tasks_status_dict[wftask_id] = wftask_status
+
+    if running_job is None:
+        # If no job is running, the chronological-last history item is also the
+        # positional-last workflow task to be included in the response.
+        if len(dataset.history) > 0:
+            last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
+        else:
+            last_valid_wftask_id = None
+    else:
+        # If a job is running, then gather more up-to-date information
+
+        # Mid priority: Set all WorkflowTask's that are part of the running job
+        # as "submitted"
+        start = running_job.first_task_index
+        end = running_job.last_task_index + 1
+        for wftask in workflow.task_list[start:end]:
+            workflow_tasks_status_dict[
+                wftask.id
+            ] = WorkflowTaskStatusTypeV2.SUBMITTED
+
+        # The last workflow task that is included in the submitted job is also
+        # the positional-last workflow task to be included in the response.
+        last_valid_wftask_id = workflow.task_list[end - 1]
+
+        # Highest priority: Read status updates coming from the running-job
+        # temporary file. Note: this file only contains information on
+        # WorkflowTask's that ran through successfully.
+        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
+        try:
+            with tmp_file.open("r") as f:
+                history = json.load(f)
+        except FileNotFoundError:
+            history = []
+        for history_item in history:
+            wftask_id = history_item["workflowtask"]["id"]
+            wftask_status = history_item["status"]
+            workflow_tasks_status_dict[wftask_id] = wftask_status
+
+    # Based on previously-gathered information, clean up the response body
+    clean_workflow_tasks_status_dict = {}
+    for wf_task in workflow.task_list:
+        wf_task_status = workflow_tasks_status_dict.get(wf_task.id)
+        if wf_task_status is None:
+            # If a wftask ID was not found, ignore it and continue
+            continue
+        clean_workflow_tasks_status_dict[wf_task.id] = wf_task_status
+        if wf_task_status == WorkflowTaskStatusTypeV2.FAILED:
+            # Starting from the beginning of `workflow.task_list`, stop the
+            # first time that you hit a failed job
+            break
+        if wf_task.id == last_valid_wftask_id:
+            # Starting from the beginning of `workflow.task_list`, stop the
+            # first time that you hit `last_valid_wftask_id``
+            break
+
+    response_body = StatusReadV2(status=clean_workflow_tasks_status_dict)
+    return response_body