fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/routes/{admin.py → admin/v1.py}

@@ -15,29 +15,29 @@ from fastapi.responses import StreamingResponse
 from sqlalchemy import func
 from sqlmodel import select
 
-from ...config import get_settings
-from ...syringe import Inject
-from ...utils import get_timestamp
-from ..db import AsyncSession
-from ..db import get_async_db
-from ..models import ApplyWorkflow
-from ..models import Dataset
-from ..models import JobStatusType
-from ..models import Project
-from ..models import Workflow
-from ..models.security import UserOAuth as User
-from ..runner._common import WORKFLOW_LOG_FILENAME
-from ..schemas import ApplyWorkflowRead
-from ..schemas import ApplyWorkflowUpdate
-from ..schemas import DatasetRead
-from ..schemas import ProjectRead
-from ..schemas import WorkflowRead
-from ..security import current_active_superuser
-from .aux._job import _write_shutdown_file
-from .aux._job import _zip_folder_to_byte_stream
-from .aux._runner import _check_backend_is_slurm
-
-router_admin = APIRouter()
+from ....config import get_settings
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import AsyncSession
+from ...db import get_async_db
+from ...models.security import UserOAuth as User
+from ...models.v1 import ApplyWorkflow
+from ...models.v1 import Dataset
+from ...models.v1 import JobStatusTypeV1
+from ...models.v1 import Project
+from ...models.v1 import Workflow
+from ...runner.filenames import WORKFLOW_LOG_FILENAME
+from ...schemas.v1 import ApplyWorkflowReadV1
+from ...schemas.v1 import ApplyWorkflowUpdateV1
+from ...schemas.v1 import DatasetReadV1
+from ...schemas.v1 import ProjectReadV1
+from ...schemas.v1 import WorkflowReadV1
+from ...security import current_active_superuser
+from ..aux._job import _write_shutdown_file
+from ..aux._job import _zip_folder_to_byte_stream
+from ..aux._runner import _check_backend_is_slurm
+
+router_admin_v1 = APIRouter()
 
 
 def _convert_to_db_timestamp(dt: datetime) -> datetime:
@@ -57,7 +57,7 @@ def _convert_to_db_timestamp(dt: datetime) -> datetime:
     return _dt
 
 
-@router_admin.get("/project/", response_model=list[ProjectRead])
+@router_admin_v1.get("/project/", response_model=list[ProjectReadV1])
 async def view_project(
     id: Optional[int] = None,
     user_id: Optional[int] = None,
@@ -65,7 +65,7 @@ async def view_project(
     timestamp_created_max: Optional[datetime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> list[ProjectRead]:
+) -> list[ProjectReadV1]:
     """
     Query `project` table.
 
@@ -95,7 +95,7 @@ async def view_project(
     return project_list
 
 
-@router_admin.get("/workflow/", response_model=list[WorkflowRead])
+@router_admin_v1.get("/workflow/", response_model=list[WorkflowReadV1])
 async def view_workflow(
     id: Optional[int] = None,
     user_id: Optional[int] = None,
@@ -105,7 +105,7 @@ async def view_workflow(
     timestamp_created_max: Optional[datetime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> list[WorkflowRead]:
+) -> list[WorkflowReadV1]:
     """
     Query `workflow` table.
 
@@ -144,7 +144,7 @@ async def view_workflow(
     return workflow_list
 
 
-@router_admin.get("/dataset/", response_model=list[DatasetRead])
+@router_admin_v1.get("/dataset/", response_model=list[DatasetReadV1])
 async def view_dataset(
     id: Optional[int] = None,
     user_id: Optional[int] = None,
@@ -155,7 +155,7 @@ async def view_dataset(
     timestamp_created_max: Optional[datetime] = None,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> list[DatasetRead]:
+) -> list[DatasetReadV1]:
     """
     Query `dataset` table.
 
@@ -197,7 +197,7 @@ async def view_dataset(
     return dataset_list
 
 
-@router_admin.get("/job/", response_model=list[ApplyWorkflowRead])
+@router_admin_v1.get("/job/", response_model=list[ApplyWorkflowReadV1])
 async def view_job(
     id: Optional[int] = None,
     user_id: Optional[int] = None,
@@ -205,7 +205,7 @@ async def view_job(
     input_dataset_id: Optional[int] = None,
     output_dataset_id: Optional[int] = None,
     workflow_id: Optional[int] = None,
-    status: Optional[JobStatusType] = None,
+    status: Optional[JobStatusTypeV1] = None,
     start_timestamp_min: Optional[datetime] = None,
     start_timestamp_max: Optional[datetime] = None,
     end_timestamp_min: Optional[datetime] = None,
@@ -213,7 +213,7 @@ async def view_job(
     log: bool = True,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> list[ApplyWorkflowRead]:
+) -> list[ApplyWorkflowReadV1]:
     """
     Query `ApplyWorkflow` table.
 
@@ -278,13 +278,13 @@ async def view_job(
     return job_list
 
 
-@router_admin.get("/job/{job_id}/", response_model=ApplyWorkflowRead)
+@router_admin_v1.get("/job/{job_id}/", response_model=ApplyWorkflowReadV1)
 async def view_single_job(
     job_id: int = None,
     show_tmp_logs: bool = False,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> ApplyWorkflowRead:
+) -> ApplyWorkflowReadV1:
 
     job = await db.get(ApplyWorkflow, job_id)
     if not job:
@@ -294,7 +294,7 @@ async def view_single_job(
         )
     await db.close()
 
-    if show_tmp_logs and (job.status == JobStatusType.SUBMITTED):
+    if show_tmp_logs and (job.status == JobStatusTypeV1.SUBMITTED):
         try:
             with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
                 job.log = f.read()
@@ -304,16 +304,16 @@ async def view_single_job(
     return job
 
 
-@router_admin.patch(
+@router_admin_v1.patch(
     "/job/{job_id}/",
-    response_model=ApplyWorkflowRead,
+    response_model=ApplyWorkflowReadV1,
 )
 async def update_job(
-    job_update: ApplyWorkflowUpdate,
+    job_update: ApplyWorkflowUpdateV1,
     job_id: int,
     user: User = Depends(current_active_superuser),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[ApplyWorkflowRead]:
+) -> Optional[ApplyWorkflowReadV1]:
     """
     Change the status of an existing job.
 
@@ -327,7 +327,7 @@ async def update_job(
             detail=f"Job {job_id} not found",
         )
 
-    if job_update.status != JobStatusType.FAILED:
+    if job_update.status != JobStatusTypeV1.FAILED:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=f"Cannot set job status to {job_update.status}",
@@ -341,7 +341,7 @@ async def update_job(
     return job
 
 
-@router_admin.get("/job/{job_id}/stop/", status_code=202)
+@router_admin_v1.get("/job/{job_id}/stop/", status_code=202)
 async def stop_job(
     job_id: int,
     user: User = Depends(current_active_superuser),
@@ -367,7 +367,7 @@ async def stop_job(
     return Response(status_code=status.HTTP_202_ACCEPTED)
 
 
-@router_admin.get(
+@router_admin_v1.get(
     "/job/{job_id}/download/",
     response_class=StreamingResponse,
 )

fractal_server/app/routes/admin/v2.py

@@ -0,0 +1,309 @@
+"""
+Definition of `/admin` routes.
+"""
+from datetime import datetime
+from datetime import timezone
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Response
+from fastapi import status
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+from sqlmodel import select
+
+from ....config import get_settings
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import AsyncSession
+from ...db import get_async_db
+from ...models.security import UserOAuth as User
+from ...models.v1 import Task
+from ...models.v2 import JobV2
+from ...models.v2 import ProjectV2
+from ...runner.filenames import WORKFLOW_LOG_FILENAME
+from ...schemas.v2 import JobReadV2
+from ...schemas.v2 import JobStatusTypeV2
+from ...schemas.v2 import JobUpdateV2
+from ...schemas.v2 import ProjectReadV2
+from ...security import current_active_superuser
+from ..aux._job import _write_shutdown_file
+from ..aux._job import _zip_folder_to_byte_stream
+from ..aux._runner import _check_backend_is_slurm
+
+router_admin_v2 = APIRouter()
+
+
+def _convert_to_db_timestamp(dt: datetime) -> datetime:
+    """
+    This function takes a timezone-aware datetime and converts it to UTC.
+    If using SQLite, it also removes the timezone information in order to make
+    the datetime comparable with datetimes in the database.
+    """
+    if dt.tzinfo is None:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"The timestamp provided has no timezone information: {dt}",
+        )
+    _dt = dt.astimezone(timezone.utc)
+    if Inject(get_settings).DB_ENGINE == "sqlite":
+        return _dt.replace(tzinfo=None)
+    return _dt
+
+
+@router_admin_v2.get("/project/", response_model=list[ProjectReadV2])
+async def view_project(
+    id: Optional[int] = None,
+    user_id: Optional[int] = None,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[ProjectReadV2]:
+    """
+    Query `ProjectV2` table.
+
+    Args:
+        id: If not `None`, select a given `project.id`.
+        user_id: If not `None`, select a given `project.user_id`.
+    """
+
+    stm = select(ProjectV2)
+
+    if id is not None:
+        stm = stm.where(ProjectV2.id == id)
+    if user_id is not None:
+        stm = stm.where(ProjectV2.user_list.any(User.id == user_id))
+
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+
+    return project_list
+
+
+@router_admin_v2.get("/job/", response_model=list[JobReadV2])
+async def view_job(
+    id: Optional[int] = None,
+    user_id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    dataset_id: Optional[int] = None,
+    workflow_id: Optional[int] = None,
+    status: Optional[JobStatusTypeV2] = None,
+    start_timestamp_min: Optional[datetime] = None,
+    start_timestamp_max: Optional[datetime] = None,
+    end_timestamp_min: Optional[datetime] = None,
+    end_timestamp_max: Optional[datetime] = None,
+    log: bool = True,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[JobReadV2]:
+    """
+    Query `ApplyWorkflow` table.
+
+    Args:
+        id: If not `None`, select a given `applyworkflow.id`.
+        project_id: If not `None`, select a given `applyworkflow.project_id`.
+        dataset_id: If not `None`, select a given
+            `applyworkflow.input_dataset_id`.
+        workflow_id: If not `None`, select a given `applyworkflow.workflow_id`.
+        status: If not `None`, select a given `applyworkflow.status`.
+        start_timestamp_min: If not `None`, select a rows with
+            `start_timestamp` after `start_timestamp_min`.
+        start_timestamp_max: If not `None`, select a rows with
+            `start_timestamp` before `start_timestamp_min`.
+        end_timestamp_min: If not `None`, select a rows with `end_timestamp`
+            after `end_timestamp_min`.
+        end_timestamp_max: If not `None`, select a rows with `end_timestamp`
+            before `end_timestamp_min`.
+        log: If `True`, include `job.log`, if `False`
+            `job.log` is set to `None`.
+    """
+    stm = select(JobV2)
+
+    if id is not None:
+        stm = stm.where(JobV2.id == id)
+    if user_id is not None:
+        stm = stm.join(ProjectV2).where(
+            ProjectV2.user_list.any(User.id == user_id)
+        )
+    if project_id is not None:
+        stm = stm.where(JobV2.project_id == project_id)
+    if dataset_id is not None:
+        stm = stm.where(JobV2.dataset_id == dataset_id)
+    if workflow_id is not None:
+        stm = stm.where(JobV2.workflow_id == workflow_id)
+    if status is not None:
+        stm = stm.where(JobV2.status == status)
+    if start_timestamp_min is not None:
+        start_timestamp_min = _convert_to_db_timestamp(start_timestamp_min)
+        stm = stm.where(JobV2.start_timestamp >= start_timestamp_min)
+    if start_timestamp_max is not None:
+        start_timestamp_max = _convert_to_db_timestamp(start_timestamp_max)
+        stm = stm.where(JobV2.start_timestamp <= start_timestamp_max)
+    if end_timestamp_min is not None:
+        end_timestamp_min = _convert_to_db_timestamp(end_timestamp_min)
+        stm = stm.where(JobV2.end_timestamp >= end_timestamp_min)
+    if end_timestamp_max is not None:
+        end_timestamp_max = _convert_to_db_timestamp(end_timestamp_max)
+        stm = stm.where(JobV2.end_timestamp <= end_timestamp_max)
+
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+    if not log:
+        for job in job_list:
+            setattr(job, "log", None)
+
+    return job_list
+
+
+@router_admin_v2.get("/job/{job_id}/", response_model=JobReadV2)
+async def view_single_job(
+    job_id: int = None,
+    show_tmp_logs: bool = False,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> JobReadV2:
+
+    job = await db.get(JobV2, job_id)
+    if not job:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+    await db.close()
+
+    if show_tmp_logs and (job.status == JobStatusTypeV2.SUBMITTED):
+        try:
+            with open(f"{job.working_dir}/{WORKFLOW_LOG_FILENAME}", "r") as f:
+                job.log = f.read()
+        except FileNotFoundError:
+            pass
+
+    return job
+
+
+@router_admin_v2.patch(
+    "/job/{job_id}/",
+    response_model=JobReadV2,
+)
+async def update_job(
+    job_update: JobUpdateV2,
+    job_id: int,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[JobReadV2]:
+    """
+    Change the status of an existing job.
+
+    This endpoint is only open to superusers, and it does not apply
+    project-based access-control to jobs.
+    """
+    job = await db.get(JobV2, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+
+    if job_update.status != JobStatusTypeV2.FAILED:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Cannot set job status to {job_update.status}",
+        )
+
+    setattr(job, "status", job_update.status)
+    setattr(job, "end_timestamp", get_timestamp())
+    await db.commit()
+    await db.refresh(job)
+    await db.close()
+    return job
+
+
+@router_admin_v2.get("/job/{job_id}/stop/", status_code=202)
+async def stop_job(
+    job_id: int,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+    """
+    Stop execution of a workflow job.
+
+    Only available for slurm backend.
+    """
+
+    _check_backend_is_slurm()
+
+    job = await db.get(JobV2, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+
+    _write_shutdown_file(job=job)
+
+    return Response(status_code=status.HTTP_202_ACCEPTED)
+
+
+@router_admin_v2.get(
+    "/job/{job_id}/download/",
+    response_class=StreamingResponse,
+)
+async def download_job_logs(
+    job_id: int,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> StreamingResponse:
+    """
+    Download job folder
+    """
+    # Get job from DB
+    job = await db.get(JobV2, job_id)
+    if job is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Job {job_id} not found",
+        )
+    # Create and return byte stream for zipped log folder
+    PREFIX_ZIP = Path(job.working_dir).name
+    zip_filename = f"{PREFIX_ZIP}_archive.zip"
+    byte_stream = _zip_folder_to_byte_stream(
+        folder=job.working_dir, zip_filename=zip_filename
+    )
+    return StreamingResponse(
+        iter([byte_stream.getvalue()]),
+        media_type="application/x-zip-compressed",
+        headers={"Content-Disposition": f"attachment;filename={zip_filename}"},
+    )
+
+
+class TaskCompatibility(BaseModel):
+    is_v2_compatible: bool
+
+
+@router_admin_v2.patch(
+    "/task-v1/{task_id}/",
+    status_code=status.HTTP_200_OK,
+)
+async def flag_task_v1_as_v2_compatible(
+    task_id: int,
+    compatibility: TaskCompatibility,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_async_db),
+) -> Response:
+
+    task = await db.get(Task, task_id)
+    if task is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Task {task_id} not found",
+        )
+
+    task.is_v2_compatible = compatibility.is_v2_compatible
+    await db.commit()
+    await db.close()
+
+    return Response(status_code=status.HTTP_200_OK)

fractal_server/app/routes/api/v1/__init__.py

@@ -14,13 +14,13 @@ from .workflowtask import router as workflowtask_router
 router_api_v1 = APIRouter()
 
 router_api_v1.include_router(
-    project_router, prefix="/project", tags=["Projects"]
+    project_router, prefix="/project", tags=["V1 Project"]
 )
-router_api_v1.include_router(task_router, prefix="/task", tags=["Tasks"])
+router_api_v1.include_router(task_router, prefix="/task", tags=["V1 Task"])
 router_api_v1.include_router(
-    taskcollection_router, prefix="/task", tags=["Task Collection"]
+    taskcollection_router, prefix="/task", tags=["V1 Task Collection"]
 )
-router_api_v1.include_router(dataset_router, tags=["Datasets"])
-router_api_v1.include_router(workflow_router, tags=["Workflows"])
-router_api_v1.include_router(workflowtask_router, tags=["Workflow Tasks"])
-router_api_v1.include_router(job_router, tags=["Jobs"])
+router_api_v1.include_router(dataset_router, tags=["V1 Dataset"])
+router_api_v1.include_router(workflow_router, tags=["V1 Workflow"])
+router_api_v1.include_router(workflowtask_router, tags=["V1 WorkflowTask"])
+router_api_v1.include_router(job_router, tags=["V1 Job"])

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -12,14 +12,14 @@ from sqlmodel import select
 from sqlmodel.sql.expression import SelectOfScalar
 
 from ....db import AsyncSession
-from ....models import ApplyWorkflow
-from ....models import Dataset
 from ....models import LinkUserProject
-from ....models import Project
-from ....models import Task
-from ....models import Workflow
-from ....models import WorkflowTask
-from ....schemas import JobStatusType
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import Dataset
+from ....models.v1 import Project
+from ....models.v1 import Task
+from ....models.v1 import Workflow
+from ....models.v1 import WorkflowTask
+from ....schemas.v1 import JobStatusTypeV1
 from ....security import User
 
 
@@ -375,7 +375,7 @@ def _get_submitted_jobs_statement() -> SelectOfScalar:
         `ApplyWorkflow.status` equal to `submitted`.
     """
     stm = select(ApplyWorkflow).where(
-        ApplyWorkflow.status == JobStatusType.SUBMITTED
+        ApplyWorkflow.status == JobStatusTypeV1.SUBMITTED
     )
     return stm