fractal-server 2.14.0a5__tar.gz → 2.14.0a7__tar.gz

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a5
+Version: 2.14.0a7
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.14.0a7/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a7"

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/routes/api/v2/task.py

@@ -152,14 +152,7 @@ async def create_task(
         db=db,
     )
 
-    if task.command_non_parallel is None:
-        task_type = "parallel"
-    elif task.command_parallel is None:
-        task_type = "non_parallel"
-    else:
-        task_type = "compound"
-
-    if task_type == "parallel" and (
+    if task.type == "parallel" and (
         task.args_schema_non_parallel is not None
         or task.meta_non_parallel is not None
     ):
@@ -170,7 +163,7 @@ async def create_task(
                 "`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
             ),
         )
-    elif task_type == "non_parallel" and (
+    elif task.type == "non_parallel" and (
         task.args_schema_parallel is not None or task.meta_parallel is not None
     ):
         raise HTTPException(
@@ -183,7 +176,7 @@ async def create_task(
 
     # Add task
 
-    db_task = TaskV2(**task.model_dump(exclude_unset=True), type=task_type)
+    db_task = TaskV2(**task.model_dump(exclude_unset=True))
     pkg_name = db_task.name
     await _verify_non_duplication_user_constraint(
         db=db, pkg_name=pkg_name, user_id=user.id, version=db_task.version

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/routes/api/v2/workflowtask.py

@@ -56,7 +56,14 @@ async def replace_workflowtask(
     )
 
     # Preliminary checks
-    if old_wftask.task_type != new_task.type:
+    EQUIVALENT_TASK_TYPES = [
+        {"non_parallel", "converter_non_parallel"},
+        {"compound", "converter_compound"},
+    ]
+    if (
+        old_wftask.task_type != new_task.type
+        and {old_wftask.task_type, new_task.type} not in EQUIVALENT_TASK_TYPES
+    ):
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
@@ -64,6 +71,7 @@ async def replace_workflowtask(
                 f"{old_wftask.task_type} to {new_task.type}."
             ),
         )
+
     if replace.args_non_parallel is not None and new_task.type == "parallel":
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -259,7 +267,10 @@ async def update_workflowtask(
                 "parallel."
             ),
         )
-    elif db_wf_task.task_type == "non_parallel" and (
+    elif db_wf_task.task_type in [
+        "non_parallel",
+        "converter_non_parallel",
+    ] and (
         workflow_task_update.args_parallel is not None
         or workflow_task_update.meta_parallel is not None
     ):

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/runner/executors/base_runner.py

@@ -1,6 +1,20 @@
 from typing import Any
 
 from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.schemas.v2.task import TaskTypeType
+
+
+TASK_TYPES_SUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "non_parallel",
+    "converter_non_parallel",
+]
+TASK_TYPES_MULTISUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "parallel",
+]
 
 
 class BaseRunner(object):
@@ -16,7 +30,7 @@ class BaseRunner(object):
         func: callable,
         parameters: dict[str, Any],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[Any, BaseException]:
         """
@@ -25,16 +39,13 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             parameters:
                 Dictionary of parameters. Must include `zarr_urls` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the init part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
@@ -43,7 +54,7 @@ class BaseRunner(object):
         func: callable,
         list_parameters: list[dict[str, Any]],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
@@ -52,33 +63,44 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             list_parameters:
                 List of dictionaries of parameters. Each one must include a
                 `zarr_url` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the compute part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
-    def validate_submit_parameters(self, parameters: dict[str, Any]) -> None:
+    def validate_submit_parameters(
+        self,
+        parameters: dict[str, Any],
+        task_type: TaskTypeType,
+    ) -> None:
         """
         Validate parameters for `submit` method
 
         Args:
             parameters: Parameters dictionary.
+            task_type: Task type.s
         """
+        if task_type not in TASK_TYPES_SUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `submit`.")
         if not isinstance(parameters, dict):
             raise ValueError("`parameters` must be a dictionary.")
-        if "zarr_urls" not in parameters.keys():
-            raise ValueError(
-                f"No 'zarr_urls' key in in {list(parameters.keys())}"
-            )
+        if task_type in ["non_parallel", "compound"]:
+            if "zarr_urls" not in parameters.keys():
+                raise ValueError(
+                    f"No 'zarr_urls' key in in {list(parameters.keys())}"
+                )
+        elif task_type in ["converter_non_parallel", "converter_compound"]:
+            if "zarr_urls" in parameters.keys():
+                raise ValueError(
+                    f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
+                )
+
         if _COMPONENT_KEY_ not in parameters.keys():
             raise ValueError(
                 f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
@@ -87,21 +109,26 @@ class BaseRunner(object):
     def validate_multisubmit_parameters(
         self,
         list_parameters: list[dict[str, Any]],
-        in_compound_task: bool,
+        task_type: TaskTypeType,
     ) -> None:
         """
         Validate parameters for `multi_submit` method
 
         Args:
             list_parameters: List of parameters dictionaries.
-            in_compound_task:
-               Whether this is the compute part of a compound task.
+            task_type: Task type.
         """
+        if task_type not in TASK_TYPES_MULTISUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `multisubmit`.")
+
+        if not isinstance(list_parameters, list):
+            raise ValueError("`parameters` must be a list.")
+
         for single_kwargs in list_parameters:
             if not isinstance(single_kwargs, dict):
-                raise RuntimeError("kwargs itemt must be a dictionary.")
+                raise ValueError("kwargs itemt must be a dictionary.")
             if "zarr_url" not in single_kwargs.keys():
-                raise RuntimeError(
+                raise ValueError(
                     f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
                 )
             if _COMPONENT_KEY_ not in single_kwargs.keys():
@@ -109,7 +136,7 @@ class BaseRunner(object):
                     f"No '{_COMPONENT_KEY_}' key "
                     f"in {list(single_kwargs.keys())}"
                 )
-        if not in_compound_task:
+        if task_type == "parallel":
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):
-                raise RuntimeError("Non-unique zarr_urls")
+                raise ValueError("Non-unique zarr_urls")

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/runner/executors/local/runner.py

@@ -9,9 +9,9 @@ from ._local_config import LocalBackendConfig
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.logger import set_logger
 
-
 logger = set_logger(__name__)
 
 
@@ -50,6 +50,7 @@ class LocalRunner(BaseRunner):
         func: callable,
         parameters: dict[str, Any],
         task_files: TaskFiles,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
@@ -61,7 +62,7 @@ class LocalRunner(BaseRunner):
             component=parameters[_COMPONENT_KEY_],
         )
 
-        self.validate_submit_parameters(parameters)
+        self.validate_submit_parameters(parameters, task_type=task_type)
         workdir_local = current_task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
@@ -83,18 +84,18 @@ class LocalRunner(BaseRunner):
         func: callable,
         list_parameters: list[dict],
         task_files: TaskFiles,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ):
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            task_type=task_type,
         )
 
         workdir_local = task_files.wftask_subfolder_local
-        if not in_compound_task:
+        if task_type not in ["compound", "converter_compound"]:
             workdir_local.mkdir()
 
         # Get local_backend_config

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -30,15 +30,11 @@ from fractal_server.app.runner.executors.slurm_common._slurm_config import (
 )
 from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
-# from fractal_server.app.history import ImageStatus
-# from fractal_server.app.history import update_all_images
-# from fractal_server.app.history import update_single_image
-# from fractal_server.app.history import update_single_image_logfile
-
 
 logger = set_logger(__name__)
 
@@ -426,7 +422,7 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ) -> tuple[Any, Exception]:
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
@@ -439,21 +435,9 @@ class RunnerSlurmSudo(BaseRunner):
         )
 
         if self.jobs != {}:
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Unexpected branch: jobs should be empty.")
 
         if self.is_shutdown():
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Cannot continue after shutdown.")
 
         # Validation phase
@@ -505,22 +489,6 @@ class RunnerSlurmSudo(BaseRunner):
                 )
             time.sleep(self.slurm_poll_interval)
 
-        if not in_compound_task:
-            if exception is None:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.DONE,
-                #     logfile=LOGFILE,
-                # )
-            else:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                #     logfile=LOGFILE,
-                # )
-
         return result, exception
 
     def multisubmit(
@@ -530,20 +498,19 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ):
         # self.scancel_jobs()
 
         self.validate_multisubmit_parameters(
-            list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            list_parameters=list_parameters, task_type=task_type
         )
 
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
 
         # Create local&remote task subfolders
-        if not in_compound_task:
+        if task_type not in ["converter_compound", "compound"]:
             original_umask = os.umask(0)
             workdir_local.mkdir(parents=True, mode=0o755)
             os.umask(original_umask)
@@ -640,28 +607,6 @@ class RunnerSlurmSudo(BaseRunner):
                     result, exception = self._postprocess_single_task(
                         task=task
                     )
-                    if not in_compound_task:
-                        pass
-                        # update_single_image_logfile(
-                        #     history_item_id=history_item_id,
-                        #     zarr_url=task.zarr_url,
-                        #     logfile=task.task_files.log_file_local,
-                        # )
-                    if not in_compound_task:
-                        if exception is None:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.DONE,
-                            # )
-                        else:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.FAILED,
-                            # )
                     if exception is None:
                         results[task.index] = result
                     else:

fractal_server-2.14.0a7/fractal_server/app/runner/v2/_db_tools.py

@@ -0,0 +1,48 @@
+from typing import Any
+
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session
+
+from fractal_server.app.models.v2 import HistoryImageCache
+
+
+def bulk_upsert_image_cache_fast(
+    *,
+    list_upsert_objects: list[dict[str, Any]],
+    db: Session,
+) -> None:
+    """
+    Insert or update many objects into `HistoryImageCache` and commit
+
+    This function is an optimized version of
+
+    ```python
+    for obj in list_upsert_objects:
+        db.merge(**obj)
+    db.commit()
+    ```
+
+    See docs at
+    https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#insert-on-conflict-upsert
+
+    FIXME: we tried to replace `index_elements` with
+    `constraint="pk_historyimagecache"`, but it did not work as expected.
+
+    Arguments:
+        list_upsert_objects:
+            List of dictionaries for objects to be upsert-ed.
+        db: A sync database session
+    """
+    if len(list_upsert_objects) == 0:
+        return None
+    stmt = pg_insert(HistoryImageCache).values(list_upsert_objects)
+    stmt = stmt.on_conflict_do_update(
+        index_elements=[
+            HistoryImageCache.zarr_url,
+            HistoryImageCache.dataset_id,
+            HistoryImageCache.workflowtask_id,
+        ],
+        set_=dict(latest_history_unit_id=stmt.excluded.latest_history_unit_id),
+    )
+    db.execute(stmt)
+    db.commit()

{fractal_server-2.14.0a5 → fractal_server-2.14.0a7}/fractal_server/app/runner/v2/runner.py

@@ -14,6 +14,8 @@ from ....images.tools import find_image_by_zarr_url
 from ..exceptions import JobExecutionError
 from .runner_functions import no_op_submit_setup_call
 from .runner_functions import run_v2_task_compound
+from .runner_functions import run_v2_task_converter_compound
+from .runner_functions import run_v2_task_converter_non_parallel
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
@@ -63,22 +65,26 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Filter images by types and attributes (in two steps)
-        type_filters = copy(current_dataset_type_filters)
-        type_filters_patch = merge_type_filters(
-            task_input_types=task.input_types,
-            wftask_type_filters=wftask.type_filters,
-        )
-        type_filters.update(type_filters_patch)
-        type_filtered_images = filter_image_list(
-            images=tmp_images,
-            type_filters=type_filters,
-            attribute_filters=None,
-        )
-        filtered_images = filter_image_list(
-            images=type_filtered_images,
-            type_filters=None,
-            attribute_filters=job_attribute_filters,
-        )
+        if wftask.task_type in ["compound", "parallel", "non_parallel"]:
+            type_filters = copy(current_dataset_type_filters)
+            type_filters_patch = merge_type_filters(
+                task_input_types=task.input_types,
+                wftask_type_filters=wftask.type_filters,
+            )
+            type_filters.update(type_filters_patch)
+            type_filtered_images = filter_image_list(
+                images=tmp_images,
+                type_filters=type_filters,
+                attribute_filters=None,
+            )
+            num_available_images = len(type_filtered_images)
+            filtered_images = filter_image_list(
+                images=type_filtered_images,
+                type_filters=None,
+                attribute_filters=job_attribute_filters,
+            )
+        else:
+            num_available_images = 0
 
         # Create history item
         with next(get_sync_db()) as db:
@@ -100,7 +106,7 @@ def execute_tasks_v2(
                 workflowtask_id=wftask.id,
                 workflowtask_dump=workflowtask_dump,
                 task_group_dump=task_group_dump,
-                num_available_images=len(type_filtered_images),
+                num_available_images=num_available_images,
                 status=XXXStatus.SUBMITTED,
             )
             db.add(history_run)
@@ -126,6 +132,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_non_parallel":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_non_parallel(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         elif task.type == "parallel":
             current_task_output, num_tasks, exceptions = run_v2_task_parallel(
                 images=filtered_images,
@@ -151,6 +173,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_compound":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_compound(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         else:
             raise ValueError(f"Unexpected error: Invalid {task.type=}.")