fractal-server 2.10.6__py3-none-any.whl → 2.11.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.10.6"
+__VERSION__ = "2.11.0"

fractal_server/app/models/v2/dataset.py

@@ -11,6 +11,7 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
+from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -41,12 +42,14 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    filters: dict[Literal["attributes", "types"], dict[str, Any]] = Field(
-        sa_column=Column(
-            JSON,
-            nullable=False,
-            server_default='{"attributes": {}, "types": {}}',
-        )
+    filters: Optional[
+        dict[Literal["attributes", "types"], dict[str, Any]]
+    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )
+    attribute_filters: AttributeFiltersType = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
     )
 
     @property

fractal_server/app/models/v2/job.py

@@ -10,6 +10,7 @@ from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
 from ...schemas.v2 import JobStatusTypeV2
+from fractal_server.images.models import AttributeFiltersType
 
 
 class JobV2(SQLModel, table=True):
@@ -49,3 +50,7 @@ class JobV2(SQLModel, table=True):
     )
     status: str = JobStatusTypeV2.SUBMITTED
     log: Optional[str] = None
+
+    attribute_filters: AttributeFiltersType = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

fractal_server/app/models/v2/workflowtask.py

@@ -25,14 +25,11 @@ class WorkflowTaskV2(SQLModel, table=True):
     args_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
     args_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
 
-    input_filters: dict[
-        Literal["attributes", "types"], dict[str, Any]
-    ] = Field(
-        sa_column=Column(
-            JSON,
-            nullable=False,
-            server_default='{"attributes": {}, "types": {}}',
-        )
+    input_filters: Optional[
+        dict[Literal["attributes", "types"], dict[str, Any]]
+    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
     )
 
     # Task

fractal_server/app/routes/api/v1/dataset.py

@@ -17,7 +17,7 @@ from ....models.v1 import ApplyWorkflow
 from ....models.v1 import Dataset
 from ....models.v1 import Project
 from ....models.v1 import Resource
-from ....runner.filenames import HISTORY_FILENAME
+from ....runner.filenames import HISTORY_FILENAME_V1
 from ....schemas.v1 import DatasetCreateV1
 from ....schemas.v1 import DatasetReadV1
 from ....schemas.v1 import DatasetStatusReadV1
@@ -511,7 +511,7 @@ async def get_workflowtask_status(
         # Highest priority: Read status updates coming from the running-job
         # temporary file. Note: this file only contains information on
         # WorkflowTask's that ran through successfully
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
+        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME_V1
         try:
             with tmp_file.open("r") as f:
                 history = json.load(f)

fractal_server/app/routes/api/v2/_aux_functions.py

@@ -21,7 +21,6 @@ from ....models.v2 import TaskV2
 from ....models.v2 import WorkflowTaskV2
 from ....models.v2 import WorkflowV2
 from ....schemas.v2 import JobStatusTypeV2
-from fractal_server.images import Filters
 
 
 async def _get_project_check_owner(
@@ -336,7 +335,7 @@ async def _workflow_insert_task(
     meta_non_parallel: Optional[dict[str, Any]] = None,
     args_non_parallel: Optional[dict[str, Any]] = None,
     args_parallel: Optional[dict[str, Any]] = None,
-    input_filters: Optional[Filters] = None,
+    type_filters: Optional[dict[str, bool]] = None,
     db: AsyncSession,
 ) -> WorkflowTaskV2:
     """
@@ -350,7 +349,7 @@ async def _workflow_insert_task(
         meta_non_parallel:
         args_non_parallel:
         args_parallel:
-        input_filters:
+        type_filters:
         db:
     """
     db_workflow = await db.get(WorkflowV2, workflow_id)
@@ -376,12 +375,6 @@ async def _workflow_insert_task(
     if final_meta_non_parallel == {}:
         final_meta_non_parallel = None
 
-    # Prepare input_filters attribute
-    if input_filters is None:
-        input_filters_kwarg = {}
-    else:
-        input_filters_kwarg = dict(input_filters=input_filters)
-
     # Create DB entry
     wf_task = WorkflowTaskV2(
         task_type=task_type,
@@ -390,7 +383,7 @@ async def _workflow_insert_task(
         args_parallel=args_parallel,
         meta_parallel=final_meta_parallel,
         meta_non_parallel=final_meta_non_parallel,
-        **input_filters_kwarg,
+        type_filters=(type_filters or dict()),
     )
     db_workflow.task_list.append(wf_task)
     flag_modified(db_workflow, "task_list")

fractal_server/app/routes/api/v2/_aux_functions_tasks.py

@@ -22,6 +22,7 @@ from fractal_server.app.routes.auth._aux_auth import (
     _verify_user_belongs_to_group,
 )
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
@@ -351,3 +352,23 @@ async def _add_warnings_to_workflow_tasks(
             wftask_data["warning"] = "Current user has no access to this task."
         wftask_list_with_warnings.append(wftask_data)
     return wftask_list_with_warnings
+
+
+def _check_type_filters_compatibility(
+    *,
+    task_input_types: dict[str, bool],
+    wftask_type_filters: dict[str, bool],
+) -> None:
+    """
+    Wrap `merge_type_filters` and raise `HTTPException` if needed.
+    """
+    try:
+        merge_type_filters(
+            task_input_types=task_input_types,
+            wftask_type_filters=wftask_type_filters,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Incompatible type filters.\nOriginal error: {str(e)}",
+        )

fractal_server/app/routes/api/v2/images.py

@@ -8,6 +8,8 @@ from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
 from pydantic import Field
+from pydantic import root_validator
+from pydantic import validator
 from sqlalchemy.orm.attributes import flag_modified
 
 from ._aux_functions import _get_dataset_check_owner
@@ -15,9 +17,14 @@ from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
-from fractal_server.images import Filters
+from fractal_server.app.schemas._filter_validators import (
+    validate_attribute_filters,
+)
+from fractal_server.app.schemas._filter_validators import validate_type_filters
+from fractal_server.app.schemas._validators import root_validate_dict_keys
 from fractal_server.images import SingleImage
 from fractal_server.images import SingleImageUpdate
+from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import match_filter
 
@@ -38,7 +45,18 @@ class ImagePage(BaseModel):
 
 class ImageQuery(BaseModel):
     zarr_url: Optional[str]
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
 
 
 @router.post(
@@ -124,7 +142,11 @@ async def query_dataset_images(
         images = [
             image
             for image in images
-            if match_filter(image, Filters(**dataset.filters))
+            if match_filter(
+                image=image,
+                type_filters=dataset.type_filters,
+                attribute_filters=dataset.attribute_filters,
+            )
         ]
 
     attributes = {}
@@ -154,13 +176,14 @@ async def query_dataset_images(
             else:
                 images = [image]
 
-        if query.filters.attributes or query.filters.types:
+        if query.attribute_filters or query.type_filters:
             images = [
                 image
                 for image in images
                 if match_filter(
-                    image,
-                    Filters(**query.filters.dict()),
+                    image=image,
+                    type_filters=query.type_filters,
+                    attribute_filters=query.attribute_filters,
                 )
             ]
 
@@ -184,7 +207,7 @@ async def query_dataset_images(
         if page > last_page:
             page = last_page
         offset = (page - 1) * page_size
-        images = images[offset : offset + page_size]  # noqa E203
+        images = images[offset : offset + page_size]
 
     return ImagePage(
         total_count=total_count,

fractal_server/app/routes/api/v2/job.py

@@ -1,4 +1,6 @@
+import asyncio
 from pathlib import Path
+from typing import Iterator
 from typing import Optional
 
 from fastapi import APIRouter
@@ -24,6 +26,14 @@ from ._aux_functions import _get_workflow_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 
+
+# https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread
+# This moves the function execution to a separate thread,
+# preventing it from blocking the event loop.
+async def zip_folder_threaded(folder: str) -> Iterator[bytes]:
+    return await asyncio.to_thread(_zip_folder_to_byte_stream_iterator, folder)
+
+
 router = APIRouter()
 
 
@@ -128,8 +138,11 @@ async def download_job_logs(
     )
     job = output["job"]
     zip_name = f"{Path(job.working_dir).name}_archive.zip"
+
+    zip_bytes_iterator = await zip_folder_threaded(job.working_dir)
+
     return StreamingResponse(
-        _zip_folder_to_byte_stream_iterator(folder=job.working_dir),
+        zip_bytes_iterator,
         media_type="application/x-zip-compressed",
         headers={"Content-Disposition": f"attachment;filename={zip_name}"},
     )

fractal_server/app/routes/api/v2/status.py

@@ -1,5 +1,3 @@
-import json
-from pathlib import Path
 from typing import Optional
 
 from fastapi import APIRouter
@@ -18,7 +16,6 @@ from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
 
 router = APIRouter()
 
@@ -98,8 +95,8 @@ async def get_workflowtask_status(
     if running_job is None:
         # If no job is running, the chronological-last history item is also the
         # positional-last workflow task to be included in the response.
-        if len(dataset.history) > 0:
-            last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
+        if len(history) > 0:
+            last_valid_wftask_id = history[-1]["workflowtask"]["id"]
         else:
             last_valid_wftask_id = None
     else:
@@ -109,7 +106,24 @@ async def get_workflowtask_status(
         # as "submitted"
         start = running_job.first_task_index
         end = running_job.last_task_index + 1
-        for wftask in workflow.task_list[start:end]:
+
+        running_job_wftasks = workflow.task_list[start:end]
+        running_job_statuses = [
+            workflow_tasks_status_dict.get(wft.id, None)
+            for wft in running_job_wftasks
+        ]
+        try:
+            first_submitted_index = running_job_statuses.index(
+                WorkflowTaskStatusTypeV2.SUBMITTED
+            )
+        except ValueError:
+            logger.warning(
+                f"Job {running_job.id} is submitted but its task list does "
+                f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
+            )
+            first_submitted_index = 0
+
+        for wftask in running_job_wftasks[first_submitted_index:]:
             workflow_tasks_status_dict[
                 wftask.id
             ] = WorkflowTaskStatusTypeV2.SUBMITTED
@@ -133,20 +147,6 @@ async def get_workflowtask_status(
             last_valid_wftask_id = None
             logger.warning(f"Now setting {last_valid_wftask_id=}.")
 
-        # Highest priority: Read status updates coming from the running-job
-        # temporary file. Note: this file only contains information on
-        # WorkflowTask's that ran through successfully.
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
-        try:
-            with tmp_file.open("r") as f:
-                history = json.load(f)
-        except FileNotFoundError:
-            history = []
-        for history_item in history:
-            wftask_id = history_item["workflowtask"]["id"]
-            wftask_status = history_item["status"]
-            workflow_tasks_status_dict[wftask_id] = wftask_status
-
     # Based on previously-gathered information, clean up the response body
     clean_workflow_tasks_status_dict = {}
     for wf_task in workflow.task_list:

fractal_server/app/routes/api/v2/submit.py

@@ -28,6 +28,7 @@ from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
+from ._aux_functions_tasks import _check_type_filters_compatibility
 from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
@@ -108,15 +109,17 @@ async def apply_workflow(
 
     # Check that tasks have read-access and are `active`
     used_task_group_ids = set()
-    for wftask in workflow.task_list[
-        first_task_index : last_task_index + 1  # noqa: E203
-    ]:
+    for wftask in workflow.task_list[first_task_index : last_task_index + 1]:
         task = await _get_task_read_access(
             user_id=user.id,
             task_id=wftask.task_id,
             require_active=True,
             db=db,
         )
+        _check_type_filters_compatibility(
+            task_input_types=task.input_types,
+            wftask_type_filters=wftask.type_filters,
+        )
         used_task_group_ids.add(task.taskgroupv2_id)
 
     # Validate user settings
@@ -159,7 +162,11 @@ async def apply_workflow(
         dataset_id=dataset_id,
         workflow_id=workflow_id,
         user_email=user.email,
-        dataset_dump=json.loads(dataset.json(exclude={"images", "history"})),
+        # The 'filters' field is not supported any more but still exists as a
+        # database column, therefore we manually exclude it from dumps.
+        dataset_dump=json.loads(
+            dataset.json(exclude={"images", "history", "filters"})
+        ),
         workflow_dump=json.loads(workflow.json(exclude={"task_list"})),
         project_dump=json.loads(project.json(exclude={"user_list"})),
         **job_create.dict(),

fractal_server/app/routes/api/v2/workflow.py

@@ -1,3 +1,4 @@
+from copy import copy
 from typing import Optional
 
 from fastapi import APIRouter
@@ -5,6 +6,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from pydantic import BaseModel
 from sqlmodel import select
 
 from ....db import AsyncSession
@@ -12,12 +14,16 @@ from ....db import get_async_db
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
+from ....runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
 from ....schemas.v2 import WorkflowCreateV2
 from ....schemas.v2 import WorkflowExportV2
 from ....schemas.v2 import WorkflowReadV2
 from ....schemas.v2 import WorkflowReadV2WithWarnings
 from ....schemas.v2 import WorkflowUpdateV2
 from ._aux_functions import _check_workflow_exists
+from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
@@ -25,6 +31,7 @@ from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.images.tools import merge_type_filters
 
 router = APIRouter()
 
@@ -284,3 +291,91 @@ async def get_user_workflows(
     res = await db.execute(stm)
     workflow_list = res.scalars().all()
     return workflow_list
+
+
+class TypeFiltersFlow(BaseModel):
+    dataset_filters: list[dict[str, bool]]
+    input_filters: list[dict[str, bool]]
+    output_filters: list[dict[str, bool]]
+
+
+@router.get(
+    "/project/{project_id}/workflow/{workflow_id}/type-filters-flow/",
+    response_model=TypeFiltersFlow,
+)
+async def get_workflow_type_filters(
+    project_id: int,
+    workflow_id: int,
+    dataset_id: Optional[int] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> Optional[WorkflowReadV2WithWarnings]:
+    """
+    Get info on an existing workflow
+    """
+
+    workflow = await _get_workflow_check_owner(
+        project_id=project_id,
+        workflow_id=workflow_id,
+        user_id=user.id,
+        db=db,
+    )
+
+    if len(workflow.task_list) == 0:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Workflow has no tasks.",
+        )
+
+    if dataset_id is None:
+        dataset_type_filters = {}
+    else:
+        res = await _get_dataset_check_owner(
+            project_id=project_id,
+            dataset_id=dataset_id,
+            user_id=user.id,
+            db=db,
+        )
+        dataset = res["dataset"]
+        dataset_type_filters = dataset.type_filters
+
+    num_tasks = len(workflow.task_list)
+    try:
+        first_task_index, last_task_index = set_start_and_last_task_index(
+            num_tasks,
+            first_task_index=first_task_index,
+            last_task_index=last_task_index,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Invalid first/last task index.\nOriginal error: {str(e)}",
+        )
+
+    list_dataset_filters = [copy(dataset_type_filters)]
+    list_filters_in = []
+    list_filters_out = []
+    for wftask in workflow.task_list[first_task_index : last_task_index + 1]:
+
+        input_type_filters = copy(dataset_type_filters)
+        patch = merge_type_filters(
+            wftask_type_filters=wftask.type_filters,
+            task_input_types=wftask.task.input_types,
+        )
+        input_type_filters.update(patch)
+        list_filters_in.append(copy(input_type_filters))
+
+        output_type_filters = wftask.task.output_types
+        list_filters_out.append(output_type_filters)
+
+        dataset_type_filters.update(wftask.task.output_types)
+        list_dataset_filters.append(copy(dataset_type_filters))
+
+    response_body = dict(
+        dataset_filters=list_dataset_filters,
+        input_filters=list_filters_in,
+        output_filters=list_filters_out,
+    )
+    return response_body

fractal_server/app/routes/api/v2/workflow_import.py

@@ -19,6 +19,7 @@ from ._aux_functions import _check_workflow_exists
 from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _add_warnings_to_workflow_tasks
+from ._aux_functions_tasks import _check_type_filters_compatibility
 from fractal_server.app.models import LinkUserGroup
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.models.v2 import TaskGroupV2
@@ -325,6 +326,13 @@ async def import_workflow(
         list_wf_tasks.append(new_wf_task)
         list_task_ids.append(task_id)
 
+    for wftask, task_id in zip(list_wf_tasks, list_task_ids):
+        task = await db.get(TaskV2, task_id)
+        _check_type_filters_compatibility(
+            task_input_types=task.input_types,
+            wftask_type_filters=wftask.type_filters,
+        )
+
     # Create new Workflow
     db_workflow = WorkflowV2(
         project_id=project_id,