fractal-server 1.4.10__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

fractal_server/app/routes/api/v2/project.py

@@ -0,0 +1,202 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlalchemy.exc import IntegrityError
+from sqlmodel import select
+
+from .....logger import close_logger
+from .....logger import set_logger
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import DatasetV2
+from ....models.v2 import JobV2
+from ....models.v2 import LinkUserProjectV2
+from ....models.v2 import ProjectV2
+from ....models.v2 import WorkflowV2
+from ....schemas.v2 import ProjectCreateV2
+from ....schemas.v2 import ProjectReadV2
+from ....schemas.v2 import ProjectUpdateV2
+from ....security import current_active_user
+from ....security import User
+from ._aux_functions import _check_project_exists
+from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
+
+router = APIRouter()
+
+
+@router.get("/project/", response_model=list[ProjectReadV2])
+async def get_list_project(
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[ProjectV2]:
+    """
+    Return list of projects user is member of
+    """
+    stm = (
+        select(ProjectV2)
+        .join(LinkUserProjectV2)
+        .where(LinkUserProjectV2.user_id == user.id)
+    )
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+    return project_list
+
+
+@router.post("/project/", response_model=ProjectReadV2, status_code=201)
+async def create_project(
+    project: ProjectCreateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Create new poject
+    """
+
+    # Check that there is no project with the same user and name
+    await _check_project_exists(
+        project_name=project.name, user_id=user.id, db=db
+    )
+
+    db_project = ProjectV2(**project.dict())
+    db_project.user_list.append(user)
+    try:
+        db.add(db_project)
+        await db.commit()
+        await db.refresh(db_project)
+        await db.close()
+    except IntegrityError as e:
+        await db.rollback()
+        logger = set_logger("create_project")
+        logger.error(str(e))
+        close_logger(logger)
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=str(e),
+        )
+
+    return db_project
+
+
+@router.get("/project/{project_id}/", response_model=ProjectReadV2)
+async def read_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[ProjectReadV2]:
+    """
+    Return info on an existing project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+    await db.close()
+    return project
+
+
+@router.patch("/project/{project_id}/", response_model=ProjectReadV2)
+async def update_project(
+    project_id: int,
+    project_update: ProjectUpdateV2,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+):
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Check that there is no project with the same user and name
+    if project_update.name is not None:
+        await _check_project_exists(
+            project_name=project_update.name, user_id=user.id, db=db
+        )
+
+    for key, value in project_update.dict(exclude_unset=True).items():
+        setattr(project, key, value)
+
+    await db.commit()
+    await db.refresh(project)
+    await db.close()
+    return project
+
+
+@router.delete("/project/{project_id}/", status_code=204)
+async def delete_project(
+    project_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete project
+    """
+    project = await _get_project_check_owner(
+        project_id=project_id, user_id=user.id, db=db
+    )
+
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current project.
+    stm = _get_submitted_jobs_statement().where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    if jobs:
+        string_ids = str([job.id for job in jobs])[1:-1]
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot delete project {project.id} because it "
+                f"is linked to active job(s) {string_ids}."
+            ),
+        )
+
+    # Cascade operations
+
+    # Workflows
+    stm = select(WorkflowV2).where(WorkflowV2.project_id == project_id)
+    res = await db.execute(stm)
+    workflows = res.scalars().all()
+    for wf in workflows:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current workflow
+        stm = select(JobV2).where(JobV2.workflow_id == wf.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.workflow_id = None
+        # Delete workflow
+        await db.delete(wf)
+    await db.commit()
+
+    # Dataset
+    stm = select(DatasetV2).where(DatasetV2.project_id == project_id)
+    res = await db.execute(stm)
+    datasets = res.scalars().all()
+    for ds in datasets:
+        # Cascade operations: set foreign-keys to null for jobs which are in
+        # relationship with the current dataset
+        stm = select(JobV2).where(JobV2.dataset_id == ds.id)
+        res = await db.execute(stm)
+        jobs = res.scalars().all()
+        for job in jobs:
+            job.dataset_id = None
+        # Delete dataset
+        await db.delete(ds)
+    await db.commit()
+
+    # Job
+    stm = select(JobV2).where(JobV2.project_id == project_id)
+    res = await db.execute(stm)
+    jobs = res.scalars().all()
+    for job in jobs:
+        job.project_id = None
+
+    await db.commit()
+
+    await db.delete(project)
+    await db.commit()
+
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v2/submit.py

@@ -0,0 +1,220 @@
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import BackgroundTasks
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import status
+from sqlmodel import select
+
+from .....config import get_settings
+from .....syringe import Inject
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v2 import JobV2
+from ....runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)  # FIXME V2
+from ....runner.v2 import submit_workflow
+from ....schemas.v2 import JobCreateV2
+from ....schemas.v2 import JobReadV2
+from ....schemas.v2 import JobStatusTypeV2
+from ....security import current_active_verified_user
+from ....security import User
+from ._aux_functions import _get_dataset_check_owner
+from ._aux_functions import _get_workflow_check_owner
+
+
+def _encode_as_utc(dt: datetime):
+    return dt.replace(tzinfo=timezone.utc).isoformat()
+
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/job/submit/",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=JobReadV2,
+)
+async def apply_workflow(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: int,
+    job_create: JobCreateV2,
+    background_tasks: BackgroundTasks,
+    user: User = Depends(current_active_verified_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[JobReadV2]:
+
+    output = await _get_dataset_check_owner(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        user_id=user.id,
+        db=db,
+    )
+    project = output["project"]
+    dataset = output["dataset"]
+
+    if dataset.read_only:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot apply workflow because dataset "
+                f"({dataset_id=}) is read_only."
+            ),
+        )
+
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+
+    if not workflow.task_list:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Workflow {workflow_id} has empty task list",
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    try:
+        first_task_index, last_task_index = set_start_and_last_task_index(
+            num_tasks,
+            first_task_index=job_create.first_task_index,
+            last_task_index=job_create.last_task_index,
+        )
+        job_create.first_task_index = first_task_index
+        job_create.last_task_index = last_task_index
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Invalid values for first_task_index or last_task_index "
+                f"(with {num_tasks=}).\n"
+                f"Original error: {str(e)}"
+            ),
+        )
+
+    # If backend is SLURM, check that the user has required attributes
+    settings = Inject(get_settings)
+    backend = settings.FRACTAL_RUNNER_BACKEND
+    if backend == "slurm":
+        if not user.slurm_user:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    f"FRACTAL_RUNNER_BACKEND={backend}, "
+                    f"but {user.slurm_user=}."
+                ),
+            )
+        if not user.cache_dir:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    f"FRACTAL_RUNNER_BACKEND={backend}, "
+                    f"but {user.cache_dir=}."
+                ),
+            )
+
+    # Check that no other job with the same dataset_id is SUBMITTED
+    stm = (
+        select(JobV2)
+        .where(JobV2.dataset_id == dataset_id)
+        .where(JobV2.status == JobStatusTypeV2.SUBMITTED)
+    )
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Dataset {dataset_id} is already in use "
+                "in submitted job(s)."
+            ),
+        )
+
+    if job_create.slurm_account is not None:
+        if job_create.slurm_account not in user.slurm_accounts:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    f"SLURM account '{job_create.slurm_account}' is not "
+                    "among those available to the current user"
+                ),
+            )
+    else:
+        if len(user.slurm_accounts) > 0:
+            job_create.slurm_account = user.slurm_accounts[0]
+
+    # Add new ApplyWorkflow object to DB
+    job = JobV2(
+        project_id=project_id,
+        dataset_id=dataset_id,
+        workflow_id=workflow_id,
+        user_email=user.email,
+        dataset_dump=dict(
+            **dataset.model_dump(exclude={"timestamp_created"}),
+            timestamp_created=_encode_as_utc(dataset.timestamp_created),
+        ),
+        workflow_dump=dict(
+            **workflow.model_dump(exclude={"task_list", "timestamp_created"}),
+            timestamp_created=_encode_as_utc(workflow.timestamp_created),
+        ),
+        project_dump=dict(
+            **project.model_dump(exclude={"user_list", "timestamp_created"}),
+            timestamp_created=_encode_as_utc(project.timestamp_created),
+        ),
+        **job_create.dict(),
+    )
+
+    # Rate Limiting:
+    # raise `429 TOO MANY REQUESTS` if this endpoint has been called with the
+    # same database keys (Project, Workflow and Datasets) during the last
+    # `settings.FRACTAL_API_SUBMIT_RATE_LIMIT` seconds.
+    stm = (
+        select(JobV2)
+        .where(JobV2.project_id == project_id)
+        .where(JobV2.workflow_id == workflow_id)
+        .where(JobV2.dataset_id == dataset_id)
+    )
+    res = await db.execute(stm)
+    db_jobs = res.scalars().all()
+    if db_jobs and any(
+        abs(
+            job.start_timestamp
+            - db_job.start_timestamp.replace(tzinfo=timezone.utc)
+        )
+        < timedelta(seconds=settings.FRACTAL_API_SUBMIT_RATE_LIMIT)
+        for db_job in db_jobs
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail=(
+                f"The endpoint 'POST /api/v2/project/{project_id}/workflow/"
+                f"{workflow_id}/apply/' "
+                "was called several times within an interval of less "
+                f"than {settings.FRACTAL_API_SUBMIT_RATE_LIMIT} seconds, using"
+                " the same foreign keys. If it was intentional, please wait "
+                "and try again."
+            ),
+        )
+
+    db.add(job)
+    await db.commit()
+    await db.refresh(job)
+
+    background_tasks.add_task(
+        submit_workflow,
+        workflow_id=workflow.id,
+        dataset_id=dataset.id,
+        job_id=job.id,
+        worker_init=job.worker_init,
+        slurm_user=user.slurm_user,
+        user_cache_dir=user.cache_dir,
+    )
+
+    await db.close()
+
+    return job

fractal_server/app/routes/api/v2/task.py

@@ -0,0 +1,222 @@
+from copy import deepcopy  # noqa
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from sqlmodel import select
+
+from .....logger import set_logger
+from ....db import AsyncSession
+from ....db import get_async_db
+from ....models.v1 import Task as TaskV1
+from ....models.v2 import TaskV2
+from ....models.v2 import WorkflowTaskV2
+from ....schemas.v2 import TaskCreateV2
+from ....schemas.v2 import TaskReadV2
+from ....schemas.v2 import TaskUpdateV2
+from ....security import current_active_user
+from ....security import current_active_verified_user
+from ....security import User
+from ._aux_functions import _get_task_check_owner
+
+router = APIRouter()
+
+logger = set_logger(__name__)
+
+
+@router.get("/", response_model=list[TaskReadV2])
+async def get_list_task(
+    args_schema_parallel: bool = True,
+    args_schema_non_parallel: bool = True,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[TaskReadV2]:
+    """
+    Get list of available tasks
+    """
+    stm = select(TaskV2)
+    res = await db.execute(stm)
+    task_list = res.scalars().all()
+    await db.close()
+    if args_schema_parallel is False:
+        for task in task_list:
+            setattr(task, "args_schema_parallel", None)
+    if args_schema_non_parallel is False:
+        for task in task_list:
+            setattr(task, "args_schema_non_parallel", None)
+
+    return task_list
+
+
+@router.get("/{task_id}/", response_model=TaskReadV2)
+async def get_task(
+    task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> TaskReadV2:
+    """
+    Get info on a specific task
+    """
+    task = await db.get(TaskV2, task_id)
+    await db.close()
+    if not task:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail="TaskV2 not found"
+        )
+    return task
+
+
+@router.patch("/{task_id}/", response_model=TaskReadV2)
+async def patch_task(
+    task_id: int,
+    task_update: TaskUpdateV2,
+    user: User = Depends(current_active_verified_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[TaskReadV2]:
+    """
+    Edit a specific task (restricted to superusers and task owner)
+    """
+
+    # Retrieve task from database
+    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
+    update = task_update.dict(exclude_unset=True)
+
+    # Forbid changes that set a previously unset command
+    if db_task.type == "non_parallel" and "command_parallel" in update:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set an unset `command_parallel`.",
+        )
+    if db_task.type == "parallel" and "command_non_parallel" in update:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Cannot set an unset `command_non_parallel`.",
+        )
+
+    for key, value in update.items():
+        setattr(db_task, key, value)
+
+    await db.commit()
+    await db.refresh(db_task)
+    await db.close()
+    return db_task
+
+
+@router.post(
+    "/", response_model=TaskReadV2, status_code=status.HTTP_201_CREATED
+)
+async def create_task(
+    task: TaskCreateV2,
+    user: User = Depends(current_active_verified_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[TaskReadV2]:
+    """
+    Create a new task
+    """
+
+    if task.command_non_parallel is None:
+        task_type = "parallel"
+    elif task.command_parallel is None:
+        task_type = "non_parallel"
+    else:
+        task_type = "compound"
+
+    if task_type == "parallel" and (
+        task.args_schema_non_parallel is not None
+        or task.meta_non_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot set `TaskV2.args_schema_non_parallel` or "
+                "`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
+            ),
+        )
+    elif task_type == "non_parallel" and (
+        task.args_schema_parallel is not None or task.meta_parallel is not None
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot set `TaskV2.args_schema_parallel` or "
+                "`TaskV2.args_schema_parallel` if TaskV2 is non_parallel"
+            ),
+        )
+
+    # Set task.owner attribute
+    if user.username:
+        owner = user.username
+    elif user.slurm_user:
+        owner = user.slurm_user
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                "Cannot add a new task because current user does not "
+                "have `username` or `slurm_user` attributes."
+            ),
+        )
+
+    # Prepend owner to task.source
+    task.source = f"{owner}:{task.source}"
+
+    # Verify that source is not already in use (note: this check is only useful
+    # to provide a user-friendly error message, but `task.source` uniqueness is
+    # already guaranteed by a constraint in the table definition).
+    stm = select(TaskV2).where(TaskV2.source == task.source)
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Source '{task.source}' already used by some TaskV2",
+        )
+    stm = select(TaskV1).where(TaskV1.source == task.source)
+    res = await db.execute(stm)
+    if res.scalars().all():
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Source '{task.source}' already used by some TaskV1",
+        )
+    # Add task
+    db_task = TaskV2(**task.dict(), owner=owner, type=task_type)
+    db.add(db_task)
+    await db.commit()
+    await db.refresh(db_task)
+    await db.close()
+    return db_task
+
+
+@router.delete("/{task_id}/", status_code=204)
+async def delete_task(
+    task_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Delete a task
+    """
+
+    db_task = await _get_task_check_owner(task_id=task_id, user=user, db=db)
+
+    # Check that the TaskV2 is not in relationship with some WorkflowTaskV2
+    stm = select(WorkflowTaskV2).filter(WorkflowTaskV2.task_id == task_id)
+    res = await db.execute(stm)
+    workflowtask_list = res.scalars().all()
+    if workflowtask_list:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Cannot remove TaskV2 {task_id} because it is currently "
+                "imported in WorkflowsV2 "
+                f"{[x.workflow_id for x in workflowtask_list]}. "
+                "If you want to remove this task, then you should first remove"
+                " the workflows.",
+            ),
+        )
+
+    await db.delete(db_task)
+    await db.commit()
+    return Response(status_code=status.HTTP_204_NO_CONTENT)