fractal-server 1.4.0a0__tar.gz → 1.4.0a1__tar.gz

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 1.4.0a0
+Version: 1.4.0a1
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-1.4.0a1/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "1.4.0a1"

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/api/__init__.py

@@ -7,6 +7,7 @@ from ...config import get_settings
 from ...syringe import Inject
 from .v1.dataset import router as dataset_router
 from .v1.job import router as job_router
+from .v1.monitoring import router as router_monitoring  # noqa
 from .v1.project import router as project_router
 from .v1.task import router as task_router
 from .v1.task_collection import router as taskcollection_router

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/api/v1/dataset.py

@@ -13,13 +13,13 @@ from ...db import AsyncSession
 from ...db import get_db
 from ...models import ApplyWorkflow
 from ...models import Dataset
-from ...models import JobStatusType
 from ...models import Resource
 from ...runner._common import HISTORY_FILENAME
 from ...schemas import DatasetCreate
 from ...schemas import DatasetRead
 from ...schemas import DatasetStatusRead
 from ...schemas import DatasetUpdate
+from ...schemas import JobStatusType
 from ...schemas import ResourceCreate
 from ...schemas import ResourceRead
 from ...schemas import ResourceUpdate

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/api/v1/job.py

@@ -7,6 +7,7 @@ from zipfile import ZipFile
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Response
 from fastapi import status
 from fastapi.responses import StreamingResponse
 from sqlmodel import select
@@ -22,11 +23,50 @@ from ...security import current_active_user
 from ...security import User
 from ._aux_functions import _get_job_check_owner
 from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_workflow_check_owner
 
 
 router = APIRouter()
 
 
+@router.get("/project/job/", response_model=list[ApplyWorkflowRead])
+async def get_user_jobs(
+    user: User = Depends(current_active_user),
+) -> list[ApplyWorkflowRead]:
+    """
+    Returns all the jobs of the current user
+    """
+
+    job_list = [
+        job for project in user.project_list for job in project.job_list
+    ]
+
+    return job_list
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/job/",
+    response_model=list[ApplyWorkflowRead],
+)
+async def get_workflow_jobs(
+    project_id: int,
+    workflow_id: int,
+    user: User = Depends(current_active_user),
+    db: AsyncSession = Depends(get_db),
+) -> Optional[list[ApplyWorkflowRead]]:
+    """
+    Returns all the jobs related to a specific workflow
+    """
+
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id, workflow_id=workflow_id, user_id=user.id, db=db
+    )
+    job_list = workflow.job_list
+    await db.close()
+
+    return job_list
+
+
 @router.get(
     "/project/{project_id}/job/{job_id}",
     response_model=ApplyWorkflowRead,
@@ -119,14 +159,14 @@ async def get_job_list(
 
 @router.get(
     "/project/{project_id}/job/{job_id}/stop/",
-    status_code=200,
+    status_code=204,
 )
 async def stop_job(
     project_id: int,
     job_id: int,
     user: User = Depends(current_active_user),
     db: AsyncSession = Depends(get_db),
-) -> Optional[ApplyWorkflow]:
+) -> Response:
     """
     Stop execution of a workflow job (only available for slurm backend)
     """
@@ -161,4 +201,4 @@ async def stop_job(
     with shutdown_file.open("w") as f:
         f.write(f"Trigger executor shutdown for {job.id=}, {project_id=}.")
 
-    return job
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server-1.4.0a1/fractal_server/app/api/v1/monitoring.py

@@ -0,0 +1,150 @@
+from datetime import datetime
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from sqlalchemy import func
+from sqlmodel import select
+
+from ...db import AsyncSession
+from ...db import get_db
+from ...models import ApplyWorkflow
+from ...models import Dataset
+from ...models import JobStatusType
+from ...models import Project
+from ...models import Workflow
+from ...schemas import ApplyWorkflowRead
+from ...schemas import DatasetRead
+from ...schemas import ProjectRead
+from ...schemas import WorkflowRead
+from ...security import current_active_superuser
+from ...security import User
+
+
+router = APIRouter()
+
+
+@router.get("/project/", response_model=list[ProjectRead])
+async def monitor_project(
+    id: Optional[int] = None,
+    user_id: Optional[int] = None,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+) -> list[ProjectRead]:
+
+    stm = select(Project)
+
+    if id is not None:
+        stm = stm.where(Project.id == id)
+
+    if user_id is not None:
+        stm = stm.where(Project.user_list.any(User.id == user_id))
+
+    res = await db.execute(stm)
+    project_list = res.scalars().all()
+    await db.close()
+
+    return project_list
+
+
+@router.get("/workflow/", response_model=list[WorkflowRead])
+async def monitor_workflow(
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    name_contains: Optional[str] = None,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+) -> list[WorkflowRead]:
+    stm = select(Workflow)
+
+    if id is not None:
+        stm = stm.where(Workflow.id == id)
+    if project_id is not None:
+        stm = stm.where(Workflow.project_id == project_id)
+    if name_contains is not None:
+        # SQLAlchemy2: use icontains
+        stm = stm.where(
+            func.lower(Workflow.name).contains(name_contains.lower())
+        )
+
+    res = await db.execute(stm)
+    workflow_list = res.scalars().all()
+    await db.close()
+
+    return workflow_list
+
+
+@router.get("/dataset/", response_model=list[DatasetRead])
+async def monitor_dataset(
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    name_contains: Optional[str] = None,
+    type: Optional[str] = None,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+) -> list[DatasetRead]:
+    stm = select(Dataset)
+
+    if id is not None:
+        stm = stm.where(Dataset.id == id)
+    if project_id is not None:
+        stm = stm.where(Dataset.project_id == project_id)
+    if name_contains is not None:
+        # SQLAlchemy2: use icontains
+        stm = stm.where(
+            func.lower(Dataset.name).contains(name_contains.lower())
+        )
+    if type is not None:
+        stm = stm.where(Dataset.type == type)
+
+    res = await db.execute(stm)
+    dataset_list = res.scalars().all()
+    await db.close()
+
+    return dataset_list
+
+
+@router.get("/job/", response_model=list[ApplyWorkflowRead])
+async def monitor_job(
+    id: Optional[int] = None,
+    project_id: Optional[int] = None,
+    input_dataset_id: Optional[int] = None,
+    output_dataset_id: Optional[int] = None,
+    workflow_id: Optional[int] = None,
+    status: Optional[JobStatusType] = None,
+    start_timestamp_min: Optional[datetime] = None,
+    start_timestamp_max: Optional[datetime] = None,
+    end_timestamp_min: Optional[datetime] = None,
+    end_timestamp_max: Optional[datetime] = None,
+    user: User = Depends(current_active_superuser),
+    db: AsyncSession = Depends(get_db),
+) -> list[ApplyWorkflowRead]:
+
+    stm = select(ApplyWorkflow)
+
+    if id is not None:
+        stm = stm.where(ApplyWorkflow.id == id)
+    if project_id is not None:
+        stm = stm.where(ApplyWorkflow.project_id == project_id)
+    if input_dataset_id is not None:
+        stm = stm.where(ApplyWorkflow.input_dataset_id == input_dataset_id)
+    if output_dataset_id is not None:
+        stm = stm.where(ApplyWorkflow.output_dataset_id == output_dataset_id)
+    if workflow_id is not None:
+        stm = stm.where(ApplyWorkflow.workflow_id == workflow_id)
+    if status is not None:
+        stm = stm.where(ApplyWorkflow.status == status)
+    if start_timestamp_min is not None:
+        stm = stm.where(ApplyWorkflow.start_timestamp >= start_timestamp_min)
+    if start_timestamp_max is not None:
+        stm = stm.where(ApplyWorkflow.start_timestamp <= start_timestamp_max)
+    if end_timestamp_min is not None:
+        stm = stm.where(ApplyWorkflow.end_timestamp >= end_timestamp_min)
+    if end_timestamp_max is not None:
+        stm = stm.where(ApplyWorkflow.end_timestamp <= end_timestamp_max)
+
+    res = await db.execute(stm)
+    job_list = res.scalars().all()
+    await db.close()
+
+    return job_list

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/api/v1/project.py

@@ -18,7 +18,6 @@ from ...db import DBSyncSession
 from ...db import get_db
 from ...db import get_sync_db
 from ...models import ApplyWorkflow
-from ...models import JobStatusType
 from ...models import LinkUserProject
 from ...models import Project
 from ...runner import submit_workflow
@@ -26,6 +25,7 @@ from ...runner import validate_workflow_compatibility
 from ...runner.common import set_start_and_last_task_index
 from ...schemas import ApplyWorkflowCreate
 from ...schemas import ApplyWorkflowRead
+from ...schemas import JobStatusType
 from ...schemas import ProjectCreate
 from ...schemas import ProjectRead
 from ...schemas import ProjectUpdate
@@ -308,7 +308,7 @@ async def apply_workflow(
         workflow_dump=dict(
             workflow.dict(exclude={"task_list"}),
             task_list=[
-                dict(wf_task.task.dict(exclude={"task"}), task=wf_task.dict())
+                dict(wf_task.dict(exclude={"task"}), task=wf_task.task.dict())
                 for wf_task in workflow.task_list
             ],
         ),

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/models/job.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from enum import Enum
 from typing import Any
 from typing import Optional
 
@@ -10,34 +9,10 @@ from sqlmodel import Field
 from sqlmodel import SQLModel
 
 from ...utils import get_timestamp
+from ..schemas import JobStatusType
 from ..schemas.applyworkflow import _ApplyWorkflowBase
 
 
-class JobStatusType(str, Enum):
-    """
-    Define the job status available
-
-    Attributes:
-        SUBMITTED:
-            The workflow has been applied but not yet scheduled with an
-            executor. In this phase, due diligence takes place, such as
-            creating working directory, assemblying arguments, etc.
-        RUNNING:
-            The workflow was scheduled with an executor. Note that it might not
-            yet be running within the executor, e.g., jobs could still be
-            pending within a SLURM executor.
-        DONE:
-            The workflow was applied successfully
-        FAILED:
-            The workflow terminated with an error.
-    """
-
-    SUBMITTED = "submitted"
-    RUNNING = "running"
-    DONE = "done"
-    FAILED = "failed"
-
-
 class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
     """
     Represent a workflow run
@@ -115,5 +90,5 @@ class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
     end_timestamp: Optional[datetime] = Field(
         default=None, sa_column=Column(DateTime(timezone=True))
     )
-    status: JobStatusType = JobStatusType.SUBMITTED
+    status: str = JobStatusType.SUBMITTED
     log: Optional[str] = None

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/runner/__init__.py

@@ -30,9 +30,9 @@ from ...utils import get_timestamp
 from ..db import DB
 from ..models import ApplyWorkflow
 from ..models import Dataset
-from ..models import JobStatusType
 from ..models import Workflow
 from ..models import WorkflowTask
+from ..schemas import JobStatusType
 from ._local import process_workflow as local_process_workflow
 from .common import close_job_logger
 from .common import JobExecutionError

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/schemas/__init__.py

@@ -3,6 +3,7 @@ Schemas for API request/response bodies
 """
 from .applyworkflow import ApplyWorkflowCreate  # noqa: F401
 from .applyworkflow import ApplyWorkflowRead  # noqa: F401
+from .applyworkflow import JobStatusType  # noqa: F401
 from .dataset import DatasetCreate  # noqa: F401
 from .dataset import DatasetRead  # noqa: F401
 from .dataset import DatasetStatusRead  # noqa: F401

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/app/schemas/applyworkflow.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from enum import Enum
 from typing import Any
 from typing import Optional
 
@@ -14,6 +15,31 @@ __all__ = (
 )
 
 
+class JobStatusType(str, Enum):
+    """
+    Define the available job statuses
+
+    Attributes:
+        SUBMITTED:
+            The workflow has been applied but not yet scheduled with an
+            executor. In this phase, due diligence takes place, such as
+            creating working directory, assemblying arguments, etc.
+        RUNNING:
+            The workflow was scheduled with an executor. Note that it might not
+            yet be running within the executor, e.g., jobs could still be
+            pending within a SLURM executor.
+        DONE:
+            The workflow was applied successfully
+        FAILED:
+            The workflow terminated with an error.
+    """
+
+    SUBMITTED = "submitted"
+    RUNNING = "running"
+    DONE = "done"
+    FAILED = "failed"
+
+
 class _ApplyWorkflowBase(BaseModel):
     """
     Base class for `ApplyWorkflow`.

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/fractal_server/main.py

@@ -48,10 +48,15 @@ def collect_routers(app: FastAPI) -> None:
     """
     from .app.api import router_default
     from .app.api import router_v1
+    from .app.api import router_monitoring
+
     from .app.security import auth_router
 
     app.include_router(router_default, prefix="/api")
     app.include_router(router_v1, prefix="/api/v1")
+    app.include_router(
+        router_monitoring, prefix="/monitoring", tags=["Monitoring"]
+    )
     app.include_router(auth_router, prefix="/auth", tags=["auth"])
 
 

{fractal_server-1.4.0a0 → fractal_server-1.4.0a1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fractal-server"
-version = "1.4.0a0"
+version = "1.4.0a1"
 description = "Server component of the Fractal analytics platform"
 authors = [
     "Jacopo Nespolo <jacopo.nespolo@exact-lab.it>",
@@ -48,7 +48,7 @@ gunicorn = ["gunicorn"]
 asgi-lifespan = "^2"
 pytest = "^7.2"
 httpx = "^0.23"
-devtools = "^0.10"
+devtools = "^0.12"
 pytest-asyncio = "^0.20"
 bumpver = "^2022.1120"
 pre-commit = "^2.19"
@@ -82,7 +82,7 @@ filterwarnings = [
 ]
 
 [tool.bumpver]
-current_version = "1.4.0a0"
+current_version = "1.4.0a1"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true
@@ -107,5 +107,5 @@ relative_files = true
 omit = ["tests/*"]
 
 [[tool.mypy.overrides]]
-module = ["devtools", "uvicorn"]
+module = ["devtools", "uvicorn", "pytest", "asgi_lifespan", "asyncpg"]
 ignore_missing_imports = true

fractal_server-1.4.0a0/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "1.4.0a0"