fractal-server 2.14.0a9__py3-none-any.whl → 2.14.0a10__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a9"
+__VERSION__ = "2.14.0a10"

fractal_server/app/models/v2/dataset.py

@@ -11,7 +11,6 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
-from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -34,20 +33,11 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
 
-    # New in V2
-
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-    attribute_filters: AttributeFiltersType = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-
     @property
     def image_zarr_urls(self) -> list[str]:
         return [image["zarr_url"] for image in self.images]

fractal_server/app/models/v2/job.py

@@ -56,3 +56,6 @@ class JobV2(SQLModel, table=True):
     attribute_filters: AttributeFiltersType = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

fractal_server/app/routes/api/v2/__init__.py

@@ -15,6 +15,7 @@ from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
 from .task_group import router as task_group_router_v2
 from .task_group_lifecycle import router as task_group_lifecycle_router_v2
+from .verify_image_types import router as verify_image_types_router
 from .workflow import router as workflow_router_v2
 from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
@@ -25,6 +26,7 @@ from fractal_server.syringe import Inject
 router_api_v2 = APIRouter()
 
 router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
+router_api_v2.include_router(verify_image_types_router, tags=["V2 Job"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])

fractal_server/app/routes/api/v2/history.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -165,6 +167,7 @@ async def get_history_run_units(
     dataset_id: int,
     workflowtask_id: int,
     history_run_id: int,
+    unit_status: Optional[HistoryUnitStatus] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
@@ -183,21 +186,23 @@ async def get_history_run_units(
     await get_history_run_or_404(history_run_id=history_run_id, db=db)
 
     # Count `HistoryUnit`s
-    res = await db.execute(
-        select(func.count(HistoryUnit.id)).where(
-            HistoryUnit.history_run_id == history_run_id
-        )
+    stmt = select(func.count(HistoryUnit.id)).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    res = await db.execute(stmt)
     total_count = res.scalar()
     page_size = pagination.page_size or total_count
 
     # Query `HistoryUnit`s
-    res = await db.execute(
-        select(HistoryUnit)
-        .where(HistoryUnit.history_run_id == history_run_id)
-        .offset((pagination.page - 1) * page_size)
-        .limit(page_size)
+    stmt = select(HistoryUnit).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    stmt = stmt.offset((pagination.page - 1) * page_size).limit(page_size)
+    res = await db.execute(stmt)
     units = res.scalars().all()
 
     return dict(

fractal_server/app/routes/api/v2/images.py

@@ -41,7 +41,6 @@ class ImagePage(PaginationResponse[SingleImage]):
 
 
 class ImageQuery(BaseModel):
-    zarr_url: Optional[str] = None
     type_filters: dict[str, bool] = Field(default_factory=dict)
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
@@ -56,6 +55,10 @@ class ImageQuery(BaseModel):
     )
 
 
+class ImageQueryWithZarrUrl(ImageQuery):
+    zarr_url: Optional[str] = None
+
+
 @router.post(
     "/project/{project_id}/dataset/{dataset_id}/images/",
     status_code=status.HTTP_201_CREATED,
@@ -115,7 +118,7 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    query: Optional[ImageQuery] = None,
+    query: Optional[ImageQueryWithZarrUrl] = None,
     pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),

fractal_server/app/routes/api/v2/submit.py

@@ -11,30 +11,32 @@ from fastapi import Request
 from fastapi import status
 from sqlmodel import select
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ....db import AsyncSession
-from ....db import get_async_db
-from ....models.v2 import JobV2
-from ....runner.set_start_and_last_task_index import (
-    set_start_and_last_task_index,
-)
-from ....runner.v2 import submit_workflow
-from ....schemas.v2 import JobCreateV2
-from ....schemas.v2 import JobReadV2
-from ....schemas.v2 import JobStatusTypeV2
-from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
 from ._aux_functions_tasks import _check_type_filters_compatibility
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
 from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
 )
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.routes.aux.validate_user_settings import (
+    validate_user_settings,
+)
+from fractal_server.app.runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
+from fractal_server.app.runner.v2 import submit_workflow
+from fractal_server.app.schemas.v2 import JobCreateV2
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 
 router = APIRouter()

fractal_server/app/routes/api/v2/verify_image_types.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import status
+
+from ._aux_functions import _get_dataset_check_owner
+from .images import ImageQuery
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.images.tools import filter_image_list
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/dataset/{dataset_id}/images/verify-unique-types/",
+    status_code=status.HTTP_200_OK,
+)
+async def verify_unique_types(
+    project_id: int,
+    dataset_id: int,
+    query: Optional[ImageQuery] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[str]:
+    # Get dataset
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+
+    # Filter images
+    if query is None:
+        filtered_images = dataset.images
+    else:
+        filtered_images = filter_image_list(
+            images=dataset.images,
+            attribute_filters=query.attribute_filters,
+            type_filters=query.type_filters,
+        )
+
+    # Get all available types (#FIXME use aux function)
+    available_types = set(
+        _type for _img in filtered_images for _type in _img["types"].keys()
+    )
+
+    # Get actual values for each available type
+    values_per_type: dict[str, set] = {
+        _type: set() for _type in available_types
+    }
+    for _img in filtered_images:
+        for _type in available_types:
+            values_per_type[_type].add(_img["types"].get(_type, False))
+
+    # Find types with non-unique value
+    non_unique_types = [
+        key for key, value in values_per_type.items() if len(value) > 1
+    ]
+    non_unique_types = sorted(non_unique_types)
+
+    return non_unique_types

fractal_server/app/routes/api/v2/workflow.py

@@ -290,6 +290,7 @@ async def get_user_workflows(
 
 
 class WorkflowTaskTypeFiltersInfo(BaseModel):
+    workflowtask_id: int
     current_type_filters: dict[str, bool]
     input_type_filters: dict[str, bool]
     output_type_filters: dict[str, bool]
@@ -301,7 +302,7 @@ async def get_workflow_type_filters(
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> dict[str, WorkflowTaskTypeFiltersInfo]:
+) -> list[WorkflowTaskTypeFiltersInfo]:
     """
     Get info on type/type-filters flow for a workflow.
     """
@@ -322,7 +323,7 @@ async def get_workflow_type_filters(
 
     current_type_filters = {}
 
-    response = {}
+    response_items = []
     for wftask in workflow.task_list:
 
         # Compute input_type_filters, based on wftask and task manifest
@@ -332,13 +333,16 @@ async def get_workflow_type_filters(
         )
 
         # Append current item to response list
-        response[str(wftask.id)] = dict(
-            current_type_filters=copy(current_type_filters),
-            input_type_filters=copy(input_type_filters),
-            output_type_filters=copy(wftask.task.output_types),
+        response_items.append(
+            dict(
+                workflowtask_id=wftask.id,
+                current_type_filters=copy(current_type_filters),
+                input_type_filters=copy(input_type_filters),
+                output_type_filters=copy(wftask.task.output_types),
+            )
         )
 
         # Update `current_type_filters`
         current_type_filters.update(wftask.task.output_types)
 
-    return response
+    return response_items

fractal_server/app/runner/executors/slurm_ssh/_check_job_status_ssh.py

@@ -0,0 +1,67 @@
+from fractal_server.app.runner.executors.slurm_common._job_states import (
+    STATES_FINISHED,
+)
+from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import FractalSSH
+
+logger = set_logger(__name__)
+
+
+def run_squeue(
+    *,
+    job_ids: list[str],
+    fractal_ssh: FractalSSH,
+) -> str:
+    job_id_single_str = ",".join([str(j) for j in job_ids])
+    cmd = (
+        f"squeue --noheader --format='%i %T' --jobs {job_id_single_str}"
+        " --states=all"
+    )
+    stdout = fractal_ssh.run_command(cmd)
+    return stdout
+
+
+def get_finished_jobs_ssh(
+    *,
+    fractal_ssh: FractalSSH,
+    job_ids: list[str],
+) -> set[str]:
+    """
+    # FIXME: make uniform with non-ssh one
+
+    Check which ones of the given Slurm jobs already finished
+
+    The function is based on the `_jobs_finished` function from
+    clusterfutures (version 0.5).
+    Original Copyright: 2022 Adrian Sampson
+    (released under the MIT licence)
+    """
+
+    # If there is no Slurm job to check, return right away
+    if not job_ids:
+        return set()
+
+    id_to_state = dict()
+
+    try:
+        stdout = run_squeue(job_ids=job_ids, fractal_ssh=fractal_ssh)
+        id_to_state = {
+            line.split()[0]: line.split()[1] for line in stdout.splitlines()
+        }
+    except Exception:  # FIXME
+        id_to_state = dict()
+        for j in job_ids:
+            try:
+                stdout = run_squeue([j])
+                id_to_state.update({stdout.split()[0]: stdout.split()[1]})
+            except Exception:
+                logger.info(f"Job {j} not found. Marked it as completed")
+                id_to_state.update({str(j): "COMPLETED"})
+
+    # Finished jobs only stay in squeue for a few mins (configurable). If
+    # a job ID isn't there, we'll assume it's finished.
+    return {
+        j
+        for j in job_ids
+        if id_to_state.get(j, "COMPLETED") in STATES_FINISHED
+    }