fractal-server 2.14.0a9__tar.gz → 2.14.0a11__tar.gz

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a9
+Version: 2.14.0a11
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a11/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a11"

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/models/v2/dataset.py

@@ -11,7 +11,6 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
-from fractal_server.images.models import AttributeFiltersType
 
 
 class DatasetV2(SQLModel, table=True):
@@ -34,20 +33,11 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(DateTime(timezone=True), nullable=False),
     )
 
-    # New in V2
-
     zarr_dir: str
     images: list[dict[str, Any]] = Field(
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    type_filters: dict[str, bool] = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-    attribute_filters: AttributeFiltersType = Field(
-        sa_column=Column(JSON, nullable=False, server_default="{}")
-    )
-
     @property
     def image_zarr_urls(self) -> list[str]:
         return [image["zarr_url"] for image in self.images]

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/models/v2/job.py

@@ -56,3 +56,6 @@ class JobV2(SQLModel, table=True):
     attribute_filters: AttributeFiltersType = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )
+    type_filters: dict[str, bool] = Field(
+        sa_column=Column(JSON, nullable=False, server_default="{}")
+    )

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/routes/api/v2/__init__.py

@@ -15,6 +15,7 @@ from .task_collection import router as task_collection_router_v2
 from .task_collection_custom import router as task_collection_router_v2_custom
 from .task_group import router as task_group_router_v2
 from .task_group_lifecycle import router as task_group_lifecycle_router_v2
+from .verify_image_types import router as verify_image_types_router
 from .workflow import router as workflow_router_v2
 from .workflow_import import router as workflow_import_router_v2
 from .workflowtask import router as workflowtask_router_v2
@@ -25,6 +26,7 @@ from fractal_server.syringe import Inject
 router_api_v2 = APIRouter()
 
 router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
+router_api_v2.include_router(verify_image_types_router, tags=["V2 Job"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/routes/api/v2/history.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -165,6 +167,7 @@ async def get_history_run_units(
     dataset_id: int,
     workflowtask_id: int,
     history_run_id: int,
+    unit_status: Optional[HistoryUnitStatus] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
     pagination: PaginationRequest = Depends(get_pagination_params),
@@ -183,21 +186,23 @@ async def get_history_run_units(
     await get_history_run_or_404(history_run_id=history_run_id, db=db)
 
     # Count `HistoryUnit`s
-    res = await db.execute(
-        select(func.count(HistoryUnit.id)).where(
-            HistoryUnit.history_run_id == history_run_id
-        )
+    stmt = select(func.count(HistoryUnit.id)).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    res = await db.execute(stmt)
     total_count = res.scalar()
     page_size = pagination.page_size or total_count
 
     # Query `HistoryUnit`s
-    res = await db.execute(
-        select(HistoryUnit)
-        .where(HistoryUnit.history_run_id == history_run_id)
-        .offset((pagination.page - 1) * page_size)
-        .limit(page_size)
+    stmt = select(HistoryUnit).where(
+        HistoryUnit.history_run_id == history_run_id
     )
+    if unit_status:
+        stmt = stmt.where(HistoryUnit.status == unit_status)
+    stmt = stmt.offset((pagination.page - 1) * page_size).limit(page_size)
+    res = await db.execute(stmt)
     units = res.scalars().all()
 
     return dict(

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/routes/api/v2/images.py

@@ -41,7 +41,6 @@ class ImagePage(PaginationResponse[SingleImage]):
 
 
 class ImageQuery(BaseModel):
-    zarr_url: Optional[str] = None
     type_filters: dict[str, bool] = Field(default_factory=dict)
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
@@ -56,6 +55,10 @@ class ImageQuery(BaseModel):
     )
 
 
+class ImageQueryWithZarrUrl(ImageQuery):
+    zarr_url: Optional[str] = None
+
+
 @router.post(
     "/project/{project_id}/dataset/{dataset_id}/images/",
     status_code=status.HTTP_201_CREATED,
@@ -115,7 +118,7 @@ async def post_new_image(
 async def query_dataset_images(
     project_id: int,
     dataset_id: int,
-    query: Optional[ImageQuery] = None,
+    query: Optional[ImageQueryWithZarrUrl] = None,
     pagination: PaginationRequest = Depends(get_pagination_params),
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/routes/api/v2/submit.py

@@ -11,30 +11,32 @@ from fastapi import Request
 from fastapi import status
 from sqlmodel import select
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ....db import AsyncSession
-from ....db import get_async_db
-from ....models.v2 import JobV2
-from ....runner.set_start_and_last_task_index import (
-    set_start_and_last_task_index,
-)
-from ....runner.v2 import submit_workflow
-from ....schemas.v2 import JobCreateV2
-from ....schemas.v2 import JobReadV2
-from ....schemas.v2 import JobStatusTypeV2
-from ...aux.validate_user_settings import validate_user_settings
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_workflow_check_owner
 from ._aux_functions import clean_app_job_list_v2
 from ._aux_functions_tasks import _check_type_filters_compatibility
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
 from fractal_server.app.models import TaskGroupV2
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.routes.api.v2._aux_functions_tasks import (
     _get_task_read_access,
 )
 from fractal_server.app.routes.auth import current_active_verified_user
+from fractal_server.app.routes.aux.validate_user_settings import (
+    validate_user_settings,
+)
+from fractal_server.app.runner.set_start_and_last_task_index import (
+    set_start_and_last_task_index,
+)
+from fractal_server.app.runner.v2.submit_workflow import submit_workflow
+from fractal_server.app.schemas.v2 import JobCreateV2
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import JobStatusTypeV2
+from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 
 router = APIRouter()

fractal_server-2.14.0a11/fractal_server/app/routes/api/v2/verify_image_types.py

@@ -0,0 +1,64 @@
+from typing import Optional
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import status
+
+from ._aux_functions import _get_dataset_check_owner
+from .images import ImageQuery
+from fractal_server.app.db import AsyncSession
+from fractal_server.app.db import get_async_db
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.routes.auth import current_active_user
+from fractal_server.images.tools import filter_image_list
+
+router = APIRouter()
+
+
+@router.post(
+    "/project/{project_id}/dataset/{dataset_id}/images/verify-unique-types/",
+    status_code=status.HTTP_200_OK,
+)
+async def verify_unique_types(
+    project_id: int,
+    dataset_id: int,
+    query: Optional[ImageQuery] = None,
+    user: UserOAuth = Depends(current_active_user),
+    db: AsyncSession = Depends(get_async_db),
+) -> list[str]:
+    # Get dataset
+    output = await _get_dataset_check_owner(
+        project_id=project_id, dataset_id=dataset_id, user_id=user.id, db=db
+    )
+    dataset = output["dataset"]
+
+    # Filter images
+    if query is None:
+        filtered_images = dataset.images
+    else:
+        filtered_images = filter_image_list(
+            images=dataset.images,
+            attribute_filters=query.attribute_filters,
+            type_filters=query.type_filters,
+        )
+
+    # Get all available types (#FIXME use aux function)
+    available_types = set(
+        _type for _img in filtered_images for _type in _img["types"].keys()
+    )
+
+    # Get actual values for each available type
+    values_per_type: dict[str, set] = {
+        _type: set() for _type in available_types
+    }
+    for _img in filtered_images:
+        for _type in available_types:
+            values_per_type[_type].add(_img["types"].get(_type, False))
+
+    # Find types with non-unique value
+    non_unique_types = [
+        key for key, value in values_per_type.items() if len(value) > 1
+    ]
+    non_unique_types = sorted(non_unique_types)
+
+    return non_unique_types

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/routes/api/v2/workflow.py

@@ -290,6 +290,7 @@ async def get_user_workflows(
 
 
 class WorkflowTaskTypeFiltersInfo(BaseModel):
+    workflowtask_id: int
     current_type_filters: dict[str, bool]
     input_type_filters: dict[str, bool]
     output_type_filters: dict[str, bool]
@@ -301,7 +302,7 @@ async def get_workflow_type_filters(
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
-) -> dict[str, WorkflowTaskTypeFiltersInfo]:
+) -> list[WorkflowTaskTypeFiltersInfo]:
     """
     Get info on type/type-filters flow for a workflow.
     """
@@ -322,7 +323,7 @@ async def get_workflow_type_filters(
 
     current_type_filters = {}
 
-    response = {}
+    response_items = []
     for wftask in workflow.task_list:
 
         # Compute input_type_filters, based on wftask and task manifest
@@ -332,13 +333,16 @@ async def get_workflow_type_filters(
         )
 
         # Append current item to response list
-        response[str(wftask.id)] = dict(
-            current_type_filters=copy(current_type_filters),
-            input_type_filters=copy(input_type_filters),
-            output_type_filters=copy(wftask.task.output_types),
+        response_items.append(
+            dict(
+                workflowtask_id=wftask.id,
+                current_type_filters=copy(current_type_filters),
+                input_type_filters=copy(input_type_filters),
+                output_type_filters=copy(wftask.task.output_types),
+            )
         )
 
         # Update `current_type_filters`
         current_type_filters.update(wftask.task.output_types)
 
-    return response
+    return response_items

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/runner/components.py

@@ -1,5 +1,2 @@
 def _index_to_component(ind: int) -> str:
     return f"{ind:07d}"
-
-
-_COMPONENT_KEY_ = "__FRACTAL_PARALLEL_COMPONENT__"

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/runner/exceptions.py

@@ -37,6 +37,10 @@ class TaskExecutionError(RuntimeError):
         self.task_name = task_name
 
 
+class TaskOutputValidationError(ValueError):
+    pass
+
+
 class JobExecutionError(RuntimeError):
     """
     Forwards errors in the execution of a task that are due to external factors

{fractal_server-2.14.0a9 → fractal_server-2.14.0a11}/fractal_server/app/runner/executors/base_runner.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.app.schemas.v2.task import TaskTypeType
 
 
@@ -29,9 +29,10 @@ class BaseRunner(object):
         self,
         func: callable,
         parameters: dict[str, Any],
-        history_item_id: int,
+        history_unit_id: int,
+        task_files: TaskFiles,
         task_type: TaskTypeType,
-        **kwargs,
+        config: Any,
     ) -> tuple[Any, BaseException]:
         """
         Run a single fractal task.
@@ -45,7 +46,7 @@ class BaseRunner(object):
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
             task_type: Task type.
-            kwargs: Runner-specific parameters.
+            config: Runner-specific parameters.
         """
         raise NotImplementedError()
 
@@ -53,9 +54,10 @@ class BaseRunner(object):
         self,
         func: callable,
         list_parameters: list[dict[str, Any]],
-        history_item_id: int,
+        history_unit_ids: list[int],
+        list_task_files: list[TaskFiles],
         task_type: TaskTypeType,
-        **kwargs,
+        config: Any,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
         Run a parallel fractal task.
@@ -70,7 +72,7 @@ class BaseRunner(object):
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
             task_type: Task type.
-            kwargs: Runner-specific parameters.
+            config: Runner-specific parameters.
         """
         raise NotImplementedError()
 
@@ -101,15 +103,11 @@ class BaseRunner(object):
                     f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
                 )
 
-        if _COMPONENT_KEY_ not in parameters.keys():
-            raise ValueError(
-                f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
-            )
-
     def validate_multisubmit_parameters(
         self,
         list_parameters: list[dict[str, Any]],
         task_type: TaskTypeType,
+        list_task_files: list[TaskFiles],
     ) -> None:
         """
         Validate parameters for `multi_submit` method
@@ -121,6 +119,12 @@ class BaseRunner(object):
         if task_type not in TASK_TYPES_MULTISUBMIT:
             raise ValueError(f"Invalid {task_type=} for `multisubmit`.")
 
+        subfolders = set(
+            task_file.wftask_subfolder_local for task_file in list_task_files
+        )
+        if len(subfolders) != 1:
+            raise ValueError(f"More than one subfolders: {subfolders}.")
+
         if not isinstance(list_parameters, list):
             raise ValueError("`parameters` must be a list.")
 
@@ -131,11 +135,6 @@ class BaseRunner(object):
                 raise ValueError(
                     f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
                 )
-            if _COMPONENT_KEY_ not in single_kwargs.keys():
-                raise ValueError(
-                    f"No '{_COMPONENT_KEY_}' key "
-                    f"in {list(single_kwargs.keys())}"
-                )
         if task_type == "parallel":
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):

fractal_server-2.14.0a9/fractal_server/app/runner/executors/local/_local_config.py → fractal_server-2.14.0a11/fractal_server/app/runner/executors/local/get_local_config.py

@@ -48,13 +48,6 @@ class LocalBackendConfig(BaseModel):
     parallel_tasks_per_job: Optional[int] = None
 
 
-def get_default_local_backend_config():
-    """
-    Return a default `LocalBackendConfig` configuration object
-    """
-    return LocalBackendConfig(parallel_tasks_per_job=None)
-
-
 def get_local_backend_config(
     wftask: WorkflowTaskV2,
     which_type: Literal["non_parallel", "parallel"],

fractal_server-2.14.0a11/fractal_server/app/runner/executors/local/runner.py

@@ -0,0 +1,215 @@
+from concurrent.futures import Future
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+from typing import Literal
+
+from .get_local_config import LocalBackendConfig
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
+
+
+class LocalRunner(BaseRunner):
+    executor: ThreadPoolExecutor
+    root_dir_local: Path
+
+    def __init__(
+        self,
+        root_dir_local: Path,
+    ):
+
+        self.root_dir_local = root_dir_local
+        self.root_dir_local.mkdir(parents=True, exist_ok=True)
+        self.executor = ThreadPoolExecutor()
+        logger.debug("Create LocalRunner")
+
+    def __enter__(self):
+        logger.debug("Enter LocalRunner")
+        return self
+
+    def shutdown(self):
+        logger.debug("Now shut LocalRunner.executor down")
+        self.executor.shutdown(
+            wait=False,
+            cancel_futures=True,
+        )
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Exit LocalRunner")
+        self.shutdown()
+        return self.executor.__exit__(exc_type, exc_val, exc_tb)
+
+    def submit(
+        self,
+        func: callable,
+        parameters: dict[str, Any],
+        history_unit_id: int,
+        task_files: TaskFiles,
+        task_type: Literal[
+            "non_parallel",
+            "converter_non_parallel",
+            "compound",
+            "converter_compound",
+        ],
+        config: LocalBackendConfig,
+    ) -> tuple[Any, Exception]:
+        logger.debug("[submit] START")
+
+        self.validate_submit_parameters(parameters, task_type=task_type)
+        workdir_local = task_files.wftask_subfolder_local
+        workdir_local.mkdir()
+
+        # SUBMISSION PHASE
+        future = self.executor.submit(
+            func,
+            parameters=parameters,
+            remote_files=task_files.remote_files_dict,
+        )
+
+        # RETRIEVAL PHASE
+        with next(get_sync_db()) as db:
+            try:
+                result = future.result()
+                logger.debug("[submit] END with result")
+                if task_type not in ["compound", "converter_compound"]:
+                    update_status_of_history_unit(
+                        history_unit_id=history_unit_id,
+                        status=HistoryUnitStatus.DONE,
+                        db_sync=db,
+                    )
+                return result, None
+            except Exception as e:
+                exception = e
+                logger.debug("[submit] END with exception")
+                update_status_of_history_unit(
+                    history_unit_id=history_unit_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
+
+                return None, exception
+
+    def multisubmit(
+        self,
+        func: callable,
+        list_parameters: list[dict],
+        history_unit_ids: list[int],
+        list_task_files: list[TaskFiles],
+        task_type: Literal["parallel", "compound", "converter_compound"],
+        config: LocalBackendConfig,
+    ):
+        """
+        Note:
+
+        1. The number of sruns and futures is equal to `len(list_parameters)`.
+        2. The number of `HistoryUnit`s is equal to `len(history_unit_ids)`.
+        3. For compound tasks, these two numbers are not the same.
+
+        For this reason, we defer database updates to the caller function,
+        when we are in one of the "compound" cases
+
+        """
+        # FIXME: De-duplicate this check
+        if task_type in ["compound", "converter_compound"]:
+            if len(history_unit_ids) != 1:
+                raise NotImplementedError(
+                    "We are breaking the assumption that compound/multisubmit "
+                    "is associated to a single HistoryUnit. This is not "
+                    "supported."
+                )
+        elif task_type == "parallel" and len(history_unit_ids) != len(
+            list_parameters
+        ):
+            raise ValueError(
+                f"{len(history_unit_ids)=} differs from "
+                f"{len(list_parameters)=}."
+            )
+
+        logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
+
+        self.validate_multisubmit_parameters(
+            list_parameters=list_parameters,
+            task_type=task_type,
+            list_task_files=list_task_files,
+        )
+
+        workdir_local = list_task_files[0].wftask_subfolder_local
+        if task_type == "parallel":
+            workdir_local.mkdir()
+
+        # Set `n_elements` and `parallel_tasks_per_job`
+        n_elements = len(list_parameters)
+        parallel_tasks_per_job = config.parallel_tasks_per_job
+        if parallel_tasks_per_job is None:
+            parallel_tasks_per_job = n_elements
+
+        # Execute tasks, in chunks of size `parallel_tasks_per_job`
+        results: dict[int, Any] = {}
+        exceptions: dict[int, BaseException] = {}
+        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+            list_parameters_chunk = list_parameters[
+                ind_chunk : ind_chunk + parallel_tasks_per_job
+            ]
+
+            active_futures: dict[int, Future] = {}
+            for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
+                positional_index = ind_chunk + ind_within_chunk
+                future = self.executor.submit(
+                    func,
+                    parameters=kwargs,
+                    remote_files=list_task_files[
+                        positional_index
+                    ].remote_files_dict,
+                )
+                active_futures[positional_index] = future
+
+            while active_futures:
+                # FIXME: add shutdown detection
+                # if file exists: cancel all futures, and raise
+                finished_futures = [
+                    index_and_future
+                    for index_and_future in active_futures.items()
+                    if not index_and_future[1].running()
+                ]
+                if len(finished_futures) == 0:
+                    continue
+
+                with next(get_sync_db()) as db:
+                    for positional_index, fut in finished_futures:
+                        active_futures.pop(positional_index)
+                        if task_type == "parallel":
+                            current_history_unit_id = history_unit_ids[
+                                positional_index
+                            ]
+
+                        try:
+                            results[positional_index] = fut.result()
+                            if task_type == "parallel":
+                                update_status_of_history_unit(
+                                    history_unit_id=current_history_unit_id,
+                                    status=HistoryUnitStatus.DONE,
+                                    db_sync=db,
+                                )
+
+                        except Exception as e:
+                            exceptions[positional_index] = e
+                            if task_type == "parallel":
+                                update_status_of_history_unit(
+                                    history_unit_id=current_history_unit_id,
+                                    status=HistoryUnitStatus.FAILED,
+                                    db_sync=db,
+                                )
+
+                            # FIXME: what should happen here? Option 1: stop
+                            # all existing tasks and shutdown runner (for the
+                            # compound-task case)
+
+        logger.debug(f"[multisubmit] END, {results=}, {exceptions=}")
+
+        return results, exceptions

{fractal_server-2.14.0a9/fractal_server/app/runner/executors/slurm_sudo → fractal_server-2.14.0a11/fractal_server/app/runner/executors/slurm_common}/_check_jobs_status.py

@@ -32,6 +32,10 @@ def run_squeue(job_ids: list[str]) -> subprocess.CompletedProcess:
     return res
 
 
+def are_all_jobs_on_squeue(job_ids: list[str]) -> bool:
+    pass
+
+
 def get_finished_jobs(job_ids: list[str]) -> set[str]:
     """
     Check which ones of the given Slurm jobs already finished