fractal-server 2.14.0a4__tar.gz → 2.14.0a6__tar.gz

{fractal_server-2.14.0a4 → fractal_server-2.14.0a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a4
+Version: 2.14.0a6
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.14.0a6/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a6"

{fractal_server-2.14.0a4 → fractal_server-2.14.0a6}/fractal_server/app/routes/api/v2/history.py

@@ -270,30 +270,39 @@ async def get_history_images(
     # FIXME reduce logging?
     prefix = f"[DS{dataset.id}-WFT{wftask.id}-images]"
 
-    # Reconstruct type filters by going through the workflow task list
-    latest_type_filters = {}
-    for current_wftask in workflow.task_list[0 : wftask.order + 1]:
-        patch = merge_type_filters(
-            wftask_type_filters=current_wftask.type_filters,
-            task_input_types=current_wftask.task.input_types,
-        )
-        latest_type_filters.update(patch)
-    logger.debug(f"{prefix} {latest_type_filters=}")
-
-    # Get all matching images from the dataset
+    # (1) Get the type-filtered list of dataset images
+
+    # (1A) Reconstruct dataset type filters by starting from {} and making
+    # incremental updates with `output_types` of all previous tasks
+    inferred_dataset_type_filters = {}
+    for current_wftask in workflow.task_list[0 : wftask.order]:
+        inferred_dataset_type_filters.update(current_wftask.task.output_types)
+    logger.debug(f"{prefix} {inferred_dataset_type_filters=}")
+    # (1B) Compute type filters for the current wftask
+    type_filters_patch = merge_type_filters(
+        task_input_types=wftask.task.input_types,
+        wftask_type_filters=wftask.type_filters,
+    )
+    logger.debug(f"{prefix} {type_filters_patch=}")
+    # (1C) Combine dataset type filters (lower priority) and current-wftask
+    # filters (higher priority)
+    actual_filters = inferred_dataset_type_filters
+    actual_filters.update(type_filters_patch)
+    logger.debug(f"{prefix} {actual_filters=}")
+    # (1D) Get all matching images from the dataset
     filtered_dataset_images = filter_image_list(
         images=dataset.images,
-        type_filters=latest_type_filters,
+        type_filters=inferred_dataset_type_filters,
     )
     logger.debug(f"{prefix} {len(dataset.images)=}")
     logger.debug(f"{prefix} {len(filtered_dataset_images)=}")
-
+    # (1E) Extract the list of URLs for filtered images
     filtered_dataset_images_url = list(
         img["zarr_url"] for img in filtered_dataset_images
     )
-    logger.debug(f"{prefix} {len(filtered_dataset_images_url)=}")
 
-    # Get pairs (zarr_url,status) for all processed images
+    # (2) Get `(zarr_url, status)` pairs for all images that have already
+    # been processed
     res = await db.execute(
         select(HistoryImageCache.zarr_url, HistoryUnit.status)
         .join(HistoryUnit)
@@ -306,8 +315,7 @@ async def get_history_images(
     list_processed_url_status = res.all()
     logger.debug(f"{prefix} {len(list_processed_url_status)=}")
 
-    # Further processing
-
+    # (3) Combine outputs from 1 and 2
     list_processed_url = list(item[0] for item in list_processed_url_status)
     logger.debug(f"{prefix} {len(list_processed_url)=}")
 

fractal_server-2.14.0a6/fractal_server/app/runner/v2/_db_tools.py

@@ -0,0 +1,48 @@
+from typing import Any
+
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session
+
+from fractal_server.app.models.v2 import HistoryImageCache
+
+
+def bulk_upsert_image_cache_fast(
+    *,
+    list_upsert_objects: list[dict[str, Any]],
+    db: Session,
+) -> None:
+    """
+    Insert or update many objects into `HistoryImageCache` and commit
+
+    This function is an optimized version of
+
+    ```python
+    for obj in list_upsert_objects:
+        db.merge(**obj)
+    db.commit()
+    ```
+
+    See docs at
+    https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#insert-on-conflict-upsert
+
+    FIXME: we tried to replace `index_elements` with
+    `constraint="pk_historyimagecache"`, but it did not work as expected.
+
+    Arguments:
+        list_upsert_objects:
+            List of dictionaries for objects to be upsert-ed.
+        db: A sync database session
+    """
+    if len(list_upsert_objects) == 0:
+        return None
+    stmt = pg_insert(HistoryImageCache).values(list_upsert_objects)
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[
+            HistoryImageCache.zarr_url,
+            HistoryImageCache.dataset_id,
+            HistoryImageCache.workflowtask_id,
+        ],
+        set_=dict(latest_history_unit_id=stmt.excluded.latest_history_unit_id),
+    )
+    db.execute(stmt)
+    db.commit()

{fractal_server-2.14.0a4 → fractal_server-2.14.0a6}/fractal_server/app/runner/v2/runner_functions.py

@@ -16,13 +16,13 @@ from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.history.status_enum import XXXStatus
-from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.components import _index_to_component
 from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.runner.v2._db_tools import bulk_upsert_image_cache_fast
 
 
 __all__ = [
@@ -114,12 +114,12 @@ def run_v2_task_non_parallel(
         which_type="non_parallel",
     )
 
-    function_kwargs = dict(
-        zarr_urls=[image["zarr_url"] for image in images],
-        zarr_dir=zarr_dir,
+    function_kwargs = {
+        "zarr_urls": [image["zarr_url"] for image in images],
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: _index_to_component(0),
         **(wftask.args_non_parallel or {}),
-    )
-    function_kwargs[_COMPONENT_KEY_] = _index_to_component(0)
+    }
 
     # Database History operations
     with next(get_sync_db()) as db:
@@ -133,16 +133,18 @@ def run_v2_task_non_parallel(
         db.commit()
         db.refresh(history_unit)
         history_unit_id = history_unit.id
-        for zarr_url in function_kwargs["zarr_urls"]:
-            db.merge(
-                HistoryImageCache(
+        bulk_upsert_image_cache_fast(
+            db=db,
+            list_upsert_objects=[
+                dict(
                     workflowtask_id=wftask.id,
                     dataset_id=dataset_id,
                     zarr_url=zarr_url,
                     latest_history_unit_id=history_unit_id,
                 )
-            )
-        db.commit()
+                for zarr_url in history_unit.zarr_urls
+            ],
+        )
 
     result, exception = executor.submit(
         functools.partial(
@@ -191,7 +193,6 @@ def run_v2_task_parallel(
     dataset_id: int,
     history_run_id: int,
 ) -> tuple[TaskOutput, int, dict[int, BaseException]]:
-
     if len(images) == 0:
         # FIXME: Do something with history units/images?
         return (TaskOutput(), 0, {})
@@ -205,39 +206,45 @@ def run_v2_task_parallel(
         which_type="parallel",
     )
 
-    list_function_kwargs = []
-    history_unit_ids = []
+    list_function_kwargs = [
+        {
+            "zarr_url": image["zarr_url"],
+            _COMPONENT_KEY_: _index_to_component(ind),
+            **(wftask.args_parallel or {}),
+        }
+        for ind, image in enumerate(images)
+    ]
+    history_units = [
+        HistoryUnit(
+            history_run_id=history_run_id,
+            status=XXXStatus.SUBMITTED,
+            logfile=None,  # FIXME
+            zarr_urls=[image["zarr_url"]],
+        )
+        for image in images
+    ]
+
     with next(get_sync_db()) as db:
-        for ind, image in enumerate(images):
-            list_function_kwargs.append(
-                dict(
-                    zarr_url=image["zarr_url"],
-                    **(wftask.args_parallel or {}),
-                ),
-            )
-            list_function_kwargs[-1][_COMPONENT_KEY_] = _index_to_component(
-                ind
-            )
-            history_unit = HistoryUnit(
-                history_run_id=history_run_id,
-                status=XXXStatus.SUBMITTED,
-                logfile=None,  # FIXME
-                zarr_urls=[image["zarr_url"]],
-            )
-            # FIXME: this should be a bulk operation
-            db.add(history_unit)
-            db.commit()
+        db.add_all(history_units)
+        db.commit()
+
+        for history_unit in history_units:
             db.refresh(history_unit)
-            db.merge(
-                HistoryImageCache(
-                    workflowtask_id=wftask.id,
-                    dataset_id=dataset_id,
-                    zarr_url=image["zarr_url"],
-                    latest_history_unit_id=history_unit.id,
-                )
+        history_unit_ids = [history_unit.id for history_unit in history_units]
+
+        history_image_caches = [
+            dict(
+                workflowtask_id=wftask.id,
+                dataset_id=dataset_id,
+                zarr_url=history_unit.zarr_urls[0],
+                latest_history_unit_id=history_unit.id,
             )
-            db.commit()
-            history_unit_ids.append(history_unit.id)
+            for history_unit in history_units
+        ]
+
+        bulk_upsert_image_cache_fast(
+            db=db, list_upsert_objects=history_image_caches
+        )
 
     results, exceptions = executor.multisubmit(
         functools.partial(
@@ -300,7 +307,6 @@ def run_v2_task_compound(
     dataset_id: int,
     history_run_id: int,
 ) -> tuple[TaskOutput, int, dict[int, BaseException]]:
-
     executor_options_init = submit_setup_call(
         wftask=wftask,
         root_dir_local=workflow_dir_local,
@@ -315,12 +321,12 @@ def run_v2_task_compound(
     )
 
     # 3/A: non-parallel init task
-    function_kwargs = dict(
-        zarr_urls=[image["zarr_url"] for image in images],
-        zarr_dir=zarr_dir,
+    function_kwargs = {
+        "zarr_urls": [image["zarr_url"] for image in images],
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: f"init_{_index_to_component(0)}",
         **(wftask.args_non_parallel or {}),
-    )
-    function_kwargs[_COMPONENT_KEY_] = f"init_{_index_to_component(0)}"
+    }
 
     # Create database History entries
     input_image_zarr_urls = function_kwargs["zarr_urls"]
@@ -337,16 +343,18 @@ def run_v2_task_compound(
         db.refresh(history_unit)
         history_unit_id = history_unit.id
         # Create one `HistoryImageCache` for each input image
-        for zarr_url in input_image_zarr_urls:
-            db.merge(
-                HistoryImageCache(
+        bulk_upsert_image_cache_fast(
+            db=db,
+            list_upsert_objects=[
+                dict(
                     workflowtask_id=wftask.id,
                     dataset_id=dataset_id,
                     zarr_url=zarr_url,
                     latest_history_unit_id=history_unit_id,
                 )
-            )
-        db.commit()
+                for zarr_url in input_image_zarr_urls
+            ],
+        )
 
     result, exception = executor.submit(
         functools.partial(
@@ -394,18 +402,15 @@ def run_v2_task_compound(
             db.commit()
         return (TaskOutput(), 0, {})
 
-    list_function_kwargs = []
-    for ind, parallelization_item in enumerate(parallelization_list):
-        list_function_kwargs.append(
-            dict(
-                zarr_url=parallelization_item.zarr_url,
-                init_args=parallelization_item.init_args,
-                **(wftask.args_parallel or {}),
-            ),
-        )
-        list_function_kwargs[-1][
-            _COMPONENT_KEY_
-        ] = f"compute_{_index_to_component(ind)}"
+    list_function_kwargs = [
+        {
+            "zarr_url": parallelization_item.zarr_url,
+            "init_args": parallelization_item.init_args,
+            _COMPONENT_KEY_: f"compute_{_index_to_component(ind)}",
+            **(wftask.args_parallel or {}),
+        }
+        for ind, parallelization_item in enumerate(parallelization_list)
+    ]
 
     results, exceptions = executor.multisubmit(
         functools.partial(

{fractal_server-2.14.0a4 → fractal_server-2.14.0a6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a4"
+version = "2.14.0a6"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a4"
+current_version = "2.14.0a6"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a4/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a4"