fractal-server 2.14.0a8__py3-none-any.whl → 2.14.0a10__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a8"
+__VERSION__ = "2.14.0a10"

fractal_server/app/models/v2/dataset.py

@@ -11,7 +11,6 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
-from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -34,20 +33,11 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
 
-    # New in V2
-
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-    attribute_filters: AttributeFiltersType = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-
     @property
     def image_zarr_urls(self) -> list[str]:
         return [image["zarr_url"] for image in self.images]

fractal_server/app/models/v2/job.py

@@ -56,3 +56,6 @@ class JobV2(SQLModel, table=True):
     attribute_filters: AttributeFiltersType = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

fractal_server/app/routes/api/v2/__init__.py

@@ -15,6 +15,7 @@ from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
 from .task_group import router as task_group_router_v2
 from .task_group_lifecycle import router as task_group_lifecycle_router_v2
+from .verify_image_types import router as verify_image_types_router
 from .workflow import router as workflow_router_v2
 from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
@@ -25,6 +26,7 @@ from fractal_server.syringe import Inject
 router_api_v2 = APIRouter()
 
 router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
+router_api_v2.include_router(verify_image_types_router, tags=["V2 Job"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])

fractal_server/app/routes/api/v2/history.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -165,6 +167,7 @@ async def get_history_run_units(
     dataset_id: int,
     workflowtask_id: int,
     history_run_id: int,
+    unit_status: Optional[HistoryUnitStatus] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
@@ -183,21 +186,23 @@ async def get_history_run_units(
     await get_history_run_or_404(history_run_id=history_run_id, db=db)
 
     # Count `HistoryUnit`s
-    res = await db.execute(
-        select(func.count(HistoryUnit.id)).where(
-            HistoryUnit.history_run_id == history_run_id
-        )
+    stmt = select(func.count(HistoryUnit.id)).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    res = await db.execute(stmt)
     total_count = res.scalar()
     page_size = pagination.page_size or total_count
 
     # Query `HistoryUnit`s
-    res = await db.execute(
-        select(HistoryUnit)
-        .where(HistoryUnit.history_run_id == history_run_id)
-        .offset((pagination.page - 1) * page_size)
-        .limit(page_size)
+    stmt = select(HistoryUnit).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    stmt = stmt.offset((pagination.page - 1) * page_size).limit(page_size)
+    res = await db.execute(stmt)
     units = res.scalars().all()
 
     return dict(

fractal_server/app/routes/api/v2/images.py

@@ -41,7 +41,6 @@ class ImagePage(PaginationResponse[SingleImage]):
 
 
 class ImageQuery(BaseModel):
-    zarr_url: Optional[str] = None
     type_filters: dict[str, bool] = Field(default_factory=dict)
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
@@ -56,6 +55,10 @@ class ImageQuery(BaseModel):
     )
 
 
+class ImageQueryWithZarrUrl(ImageQuery):
+    zarr_url: Optional[str] = None
+
+
 @router.post(
     "/project/{project_id}/dataset/{dataset_id}/images/",
     status_code=status.HTTP_201_CREATED,
@@ -115,7 +118,7 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    query: Optional[ImageQuery] = None,
+    query: Optional[ImageQueryWithZarrUrl] = None,
     pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),

fractal_server/app/routes/api/v2/submit.py

@@ -11,30 +11,32 @@ from fastapi import Request
 from fastapi import status
 from sqlmodel import select
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ....db import AsyncSession
-from ....db import get_async_db
-from ....models.v2 import JobV2
-from ....runner.set_start_and_last_task_index import (
-    set_start_and_last_task_index,
-)
-from ....runner.v2 import submit_workflow
-from ....schemas.v2 import JobCreateV2
-from ....schemas.v2 import JobReadV2
-from ....schemas.v2 import JobStatusTypeV2
-from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
 from ._aux_functions_tasks import _check_type_filters_compatibility
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
 from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
 )
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.routes.aux.validate_user_settings import (
+    validate_user_settings,
+)
+from fractal_server.app.runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
+from fractal_server.app.runner.v2 import submit_workflow
+from fractal_server.app.schemas.v2 import JobCreateV2
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 
 router = APIRouter()

fractal_server/app/routes/api/v2/verify_image_types.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import status
+
+from ._aux_functions import _get_dataset_check_owner
+from .images import ImageQuery
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.images.tools import filter_image_list
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/dataset/{dataset_id}/images/verify-unique-types/",
+    status_code=status.HTTP_200_OK,
+)
+async def verify_unique_types(
+    project_id: int,
+    dataset_id: int,
+    query: Optional[ImageQuery] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[str]:
+    # Get dataset
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+
+    # Filter images
+    if query is None:
+        filtered_images = dataset.images
+    else:
+        filtered_images = filter_image_list(
+            images=dataset.images,
+            attribute_filters=query.attribute_filters,
+            type_filters=query.type_filters,
+        )
+
+    # Get all available types (#FIXME use aux function)
+    available_types = set(
+        _type for _img in filtered_images for _type in _img["types"].keys()
+    )
+
+    # Get actual values for each available type
+    values_per_type: dict[str, set] = {
+        _type: set() for _type in available_types
+    }
+    for _img in filtered_images:
+        for _type in available_types:
+            values_per_type[_type].add(_img["types"].get(_type, False))
+
+    # Find types with non-unique value
+    non_unique_types = [
+        key for key, value in values_per_type.items() if len(value) > 1
+    ]
+    non_unique_types = sorted(non_unique_types)
+
+    return non_unique_types

fractal_server/app/routes/api/v2/workflow.py

@@ -14,16 +14,12 @@ from ....db import get_async_db
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
-from ....runner.set_start_and_last_task_index import (
-    set_start_and_last_task_index,
-)
 from ....schemas.v2 import WorkflowCreateV2
 from ....schemas.v2 import WorkflowExportV2
 from ....schemas.v2 import WorkflowReadV2
 from ....schemas.v2 import WorkflowReadV2WithWarnings
 from ....schemas.v2 import WorkflowUpdateV2
 from ._aux_functions import _check_workflow_exists
-from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
@@ -293,27 +289,22 @@ async def get_user_workflows(
     return workflow_list
 
 
-class TypeFiltersFlow(BaseModel):
-    dataset_filters: list[dict[str, bool]]
-    input_filters: list[dict[str, bool]]
-    output_filters: list[dict[str, bool]]
+class WorkflowTaskTypeFiltersInfo(BaseModel):
+    workflowtask_id: int
+    current_type_filters: dict[str, bool]
+    input_type_filters: dict[str, bool]
+    output_type_filters: dict[str, bool]
 
 
-@router.get(
-    "/project/{project_id}/workflow/{workflow_id}/type-filters-flow/",
-    response_model=TypeFiltersFlow,
-)
+@router.get("/project/{project_id}/workflow/{workflow_id}/type-filters-flow/")
 async def get_workflow_type_filters(
     project_id: int,
     workflow_id: int,
-    dataset_id: Optional[int] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> Optional[WorkflowReadV2WithWarnings]:
+) -> list[WorkflowTaskTypeFiltersInfo]:
     """
-    Get info on an existing workflow
+    Get info on type/type-filters flow for a workflow.
     """
 
     workflow = await _get_workflow_check_owner(
@@ -323,59 +314,35 @@ async def get_workflow_type_filters(
         db=db,
     )
 
-    if len(workflow.task_list) == 0:
+    num_tasks = len(workflow.task_list)
+    if num_tasks == 0:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail="Workflow has no tasks.",
         )
 
-    if dataset_id is None:
-        dataset_type_filters = {}
-    else:
-        res = await _get_dataset_check_owner(
-            project_id=project_id,
-            dataset_id=dataset_id,
-            user_id=user.id,
-            db=db,
-        )
-        dataset = res["dataset"]
-        dataset_type_filters = dataset.type_filters
-
-    num_tasks = len(workflow.task_list)
-    try:
-        first_task_index, last_task_index = set_start_and_last_task_index(
-            num_tasks,
-            first_task_index=first_task_index,
-            last_task_index=last_task_index,
-        )
-    except ValueError as e:
-        raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail=f"Invalid first/last task index.\nOriginal error: {str(e)}",
-        )
+    current_type_filters = {}
 
-    list_dataset_filters = [copy(dataset_type_filters)]
-    list_filters_in = []
-    list_filters_out = []
-    for wftask in workflow.task_list[first_task_index : last_task_index + 1]:
+    response_items = []
+    for wftask in workflow.task_list:
 
-        input_type_filters = copy(dataset_type_filters)
-        patch = merge_type_filters(
+        # Compute input_type_filters, based on wftask and task manifest
+        input_type_filters = merge_type_filters(
             wftask_type_filters=wftask.type_filters,
             task_input_types=wftask.task.input_types,
         )
-        input_type_filters.update(patch)
-        list_filters_in.append(copy(input_type_filters))
 
-        output_type_filters = wftask.task.output_types
-        list_filters_out.append(output_type_filters)
+        # Append current item to response list
+        response_items.append(
+            dict(
+                workflowtask_id=wftask.id,
+                current_type_filters=copy(current_type_filters),
+                input_type_filters=copy(input_type_filters),
+                output_type_filters=copy(wftask.task.output_types),
+            )
+        )
 
-        dataset_type_filters.update(wftask.task.output_types)
-        list_dataset_filters.append(copy(dataset_type_filters))
+        # Update `current_type_filters`
+        current_type_filters.update(wftask.task.output_types)
 
-    response_body = dict(
-        dataset_filters=list_dataset_filters,
-        input_filters=list_filters_in,
-        output_filters=list_filters_out,
-    )
-    return response_body
+    return response_items

fractal_server/app/runner/executors/slurm_ssh/_check_job_status_ssh.py

@@ -0,0 +1,67 @@
+from fractal_server.app.runner.executors.slurm_common._job_states import (
+    STATES_FINISHED,
+)
+from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import FractalSSH
+
+logger = set_logger(__name__)
+
+
+def run_squeue(
+    *,
+    job_ids: list[str],
+    fractal_ssh: FractalSSH,
+) -> str:
+    job_id_single_str = ",".join([str(j) for j in job_ids])
+    cmd = (
+        f"squeue --noheader --format='%i %T' --jobs {job_id_single_str}"
+        " --states=all"
+    )
+    stdout = fractal_ssh.run_command(cmd)
+    return stdout
+
+
+def get_finished_jobs_ssh(
+    *,
+    fractal_ssh: FractalSSH,
+    job_ids: list[str],
+) -> set[str]:
+    """
+    # FIXME: make uniform with non-ssh one
+
+    Check which ones of the given Slurm jobs already finished
+
+    The function is based on the `_jobs_finished` function from
+    clusterfutures (version 0.5).
+    Original Copyright: 2022 Adrian Sampson
+    (released under the MIT licence)
+    """
+
+    # If there is no Slurm job to check, return right away
+    if not job_ids:
+        return set()
+
+    id_to_state = dict()
+
+    try:
+        stdout = run_squeue(job_ids=job_ids, fractal_ssh=fractal_ssh)
+        id_to_state = {
+            line.split()[0]: line.split()[1] for line in stdout.splitlines()
+        }
+    except Exception:  # FIXME
+        id_to_state = dict()
+        for j in job_ids:
+            try:
+                stdout = run_squeue([j])
+                id_to_state.update({stdout.split()[0]: stdout.split()[1]})
+            except Exception:
+                logger.info(f"Job {j} not found. Marked it as completed")
+                id_to_state.update({str(j): "COMPLETED"})
+
+    # Finished jobs only stay in squeue for a few mins (configurable). If
+    # a job ID isn't there, we'll assume it's finished.
+    return {
+        j
+        for j in job_ids
+        if id_to_state.get(j, "COMPLETED") in STATES_FINISHED
+    }