fractal-server 2.14.0a6__py3-none-any.whl → 2.14.0a7__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a6"
+__VERSION__ = "2.14.0a7"

fractal_server/app/routes/api/v2/task.py

@@ -152,14 +152,7 @@ async def create_task(
         db=db,
     )
 
-    if task.command_non_parallel is None:
-        task_type = "parallel"
-    elif task.command_parallel is None:
-        task_type = "non_parallel"
-    else:
-        task_type = "compound"
-
-    if task_type == "parallel" and (
+    if task.type == "parallel" and (
         task.args_schema_non_parallel is not None
         or task.meta_non_parallel is not None
     ):
@@ -170,7 +163,7 @@ async def create_task(
                 "`TaskV2.args_schema_non_parallel` if TaskV2 is parallel"
             ),
         )
-    elif task_type == "non_parallel" and (
+    elif task.type == "non_parallel" and (
         task.args_schema_parallel is not None or task.meta_parallel is not None
     ):
         raise HTTPException(
@@ -183,7 +176,7 @@ async def create_task(
 
     # Add task
 
-    db_task = TaskV2(**task.model_dump(exclude_unset=True), type=task_type)
+    db_task = TaskV2(**task.model_dump(exclude_unset=True))
     pkg_name = db_task.name
     await _verify_non_duplication_user_constraint(
         db=db, pkg_name=pkg_name, user_id=user.id, version=db_task.version

fractal_server/app/routes/api/v2/workflowtask.py

@@ -56,7 +56,14 @@ async def replace_workflowtask(
     )
 
     # Preliminary checks
-    if old_wftask.task_type != new_task.type:
+    EQUIVALENT_TASK_TYPES = [
+        {"non_parallel", "converter_non_parallel"},
+        {"compound", "converter_compound"},
+    ]
+    if (
+        old_wftask.task_type != new_task.type
+        and {old_wftask.task_type, new_task.type} not in EQUIVALENT_TASK_TYPES
+    ):
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
             detail=(
@@ -64,6 +71,7 @@ async def replace_workflowtask(
                 f"{old_wftask.task_type} to {new_task.type}."
             ),
         )
+
     if replace.args_non_parallel is not None and new_task.type == "parallel":
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
@@ -259,7 +267,10 @@ async def update_workflowtask(
                 "parallel."
             ),
         )
-    elif db_wf_task.task_type == "non_parallel" and (
+    elif db_wf_task.task_type in [
+        "non_parallel",
+        "converter_non_parallel",
+    ] and (
         workflow_task_update.args_parallel is not None
         or workflow_task_update.meta_parallel is not None
     ):

fractal_server/app/runner/executors/base_runner.py

@@ -1,6 +1,20 @@
 from typing import Any
 
 from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.schemas.v2.task import TaskTypeType
+
+
+TASK_TYPES_SUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "non_parallel",
+    "converter_non_parallel",
+]
+TASK_TYPES_MULTISUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "parallel",
+]
 
 
 class BaseRunner(object):
@@ -16,7 +30,7 @@ class BaseRunner(object):
         func: callable,
         parameters: dict[str, Any],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[Any, BaseException]:
         """
@@ -25,16 +39,13 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             parameters:
                 Dictionary of parameters. Must include `zarr_urls` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the init part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
@@ -43,7 +54,7 @@ class BaseRunner(object):
         func: callable,
         list_parameters: list[dict[str, Any]],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
@@ -52,33 +63,44 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             list_parameters:
                 List of dictionaries of parameters. Each one must include a
                 `zarr_url` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the compute part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
-    def validate_submit_parameters(self, parameters: dict[str, Any]) -> None:
+    def validate_submit_parameters(
+        self,
+        parameters: dict[str, Any],
+        task_type: TaskTypeType,
+    ) -> None:
         """
         Validate parameters for `submit` method
 
         Args:
             parameters: Parameters dictionary.
+            task_type: Task type.s
         """
+        if task_type not in TASK_TYPES_SUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `submit`.")
         if not isinstance(parameters, dict):
             raise ValueError("`parameters` must be a dictionary.")
-        if "zarr_urls" not in parameters.keys():
-            raise ValueError(
-                f"No 'zarr_urls' key in in {list(parameters.keys())}"
-            )
+        if task_type in ["non_parallel", "compound"]:
+            if "zarr_urls" not in parameters.keys():
+                raise ValueError(
+                    f"No 'zarr_urls' key in in {list(parameters.keys())}"
+                )
+        elif task_type in ["converter_non_parallel", "converter_compound"]:
+            if "zarr_urls" in parameters.keys():
+                raise ValueError(
+                    f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
+                )
+
         if _COMPONENT_KEY_ not in parameters.keys():
             raise ValueError(
                 f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
@@ -87,21 +109,26 @@ class BaseRunner(object):
     def validate_multisubmit_parameters(
         self,
         list_parameters: list[dict[str, Any]],
-        in_compound_task: bool,
+        task_type: TaskTypeType,
     ) -> None:
         """
         Validate parameters for `multi_submit` method
 
         Args:
             list_parameters: List of parameters dictionaries.
-            in_compound_task:
-               Whether this is the compute part of a compound task.
+            task_type: Task type.
         """
+        if task_type not in TASK_TYPES_MULTISUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `multisubmit`.")
+
+        if not isinstance(list_parameters, list):
+            raise ValueError("`parameters` must be a list.")
+
         for single_kwargs in list_parameters:
             if not isinstance(single_kwargs, dict):
-                raise RuntimeError("kwargs itemt must be a dictionary.")
+                raise ValueError("kwargs itemt must be a dictionary.")
             if "zarr_url" not in single_kwargs.keys():
-                raise RuntimeError(
+                raise ValueError(
                     f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
                 )
             if _COMPONENT_KEY_ not in single_kwargs.keys():
@@ -109,7 +136,7 @@ class BaseRunner(object):
                     f"No '{_COMPONENT_KEY_}' key "
                     f"in {list(single_kwargs.keys())}"
                 )
-        if not in_compound_task:
+        if task_type == "parallel":
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):
-                raise RuntimeError("Non-unique zarr_urls")
+                raise ValueError("Non-unique zarr_urls")

fractal_server/app/runner/executors/local/runner.py

@@ -9,9 +9,9 @@ from ._local_config import LocalBackendConfig
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.logger import set_logger
 
-
 logger = set_logger(__name__)
 
 
@@ -50,6 +50,7 @@ class LocalRunner(BaseRunner):
         func: callable,
         parameters: dict[str, Any],
         task_files: TaskFiles,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
@@ -61,7 +62,7 @@ class LocalRunner(BaseRunner):
             component=parameters[_COMPONENT_KEY_],
         )
 
-        self.validate_submit_parameters(parameters)
+        self.validate_submit_parameters(parameters, task_type=task_type)
         workdir_local = current_task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
@@ -83,18 +84,18 @@ class LocalRunner(BaseRunner):
         func: callable,
         list_parameters: list[dict],
         task_files: TaskFiles,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ):
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            task_type=task_type,
         )
 
         workdir_local = task_files.wftask_subfolder_local
-        if not in_compound_task:
+        if task_type not in ["compound", "converter_compound"]:
             workdir_local.mkdir()
 
         # Get local_backend_config

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -30,15 +30,11 @@ from fractal_server.app.runner.executors.slurm_common._slurm_config import (
 )
 from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
-# from fractal_server.app.history import ImageStatus
-# from fractal_server.app.history import update_all_images
-# from fractal_server.app.history import update_single_image
-# from fractal_server.app.history import update_single_image_logfile
-
 
 logger = set_logger(__name__)
 
@@ -426,7 +422,7 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ) -> tuple[Any, Exception]:
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
@@ -439,21 +435,9 @@ class RunnerSlurmSudo(BaseRunner):
         )
 
         if self.jobs != {}:
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Unexpected branch: jobs should be empty.")
 
         if self.is_shutdown():
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Cannot continue after shutdown.")
 
         # Validation phase
@@ -505,22 +489,6 @@ class RunnerSlurmSudo(BaseRunner):
                 )
             time.sleep(self.slurm_poll_interval)
 
-        if not in_compound_task:
-            if exception is None:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.DONE,
-                #     logfile=LOGFILE,
-                # )
-            else:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                #     logfile=LOGFILE,
-                # )
-
         return result, exception
 
     def multisubmit(
@@ -530,20 +498,19 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ):
         # self.scancel_jobs()
 
         self.validate_multisubmit_parameters(
-            list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            list_parameters=list_parameters, task_type=task_type
         )
 
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
 
         # Create local&remote task subfolders
-        if not in_compound_task:
+        if task_type not in ["converter_compound", "compound"]:
             original_umask = os.umask(0)
             workdir_local.mkdir(parents=True, mode=0o755)
             os.umask(original_umask)
@@ -640,28 +607,6 @@ class RunnerSlurmSudo(BaseRunner):
                     result, exception = self._postprocess_single_task(
                         task=task
                     )
-                    if not in_compound_task:
-                        pass
-                        # update_single_image_logfile(
-                        #     history_item_id=history_item_id,
-                        #     zarr_url=task.zarr_url,
-                        #     logfile=task.task_files.log_file_local,
-                        # )
-                    if not in_compound_task:
-                        if exception is None:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.DONE,
-                            # )
-                        else:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.FAILED,
-                            # )
                     if exception is None:
                         results[task.index] = result
                     else:

fractal_server/app/runner/v2/runner.py

@@ -14,6 +14,8 @@ from ....images.tools import find_image_by_zarr_url
 from ..exceptions import JobExecutionError
 from .runner_functions import no_op_submit_setup_call
 from .runner_functions import run_v2_task_compound
+from .runner_functions import run_v2_task_converter_compound
+from .runner_functions import run_v2_task_converter_non_parallel
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
@@ -63,22 +65,26 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Filter images by types and attributes (in two steps)
-        type_filters = copy(current_dataset_type_filters)
-        type_filters_patch = merge_type_filters(
-            task_input_types=task.input_types,
-            wftask_type_filters=wftask.type_filters,
-        )
-        type_filters.update(type_filters_patch)
-        type_filtered_images = filter_image_list(
-            images=tmp_images,
-            type_filters=type_filters,
-            attribute_filters=None,
-        )
-        filtered_images = filter_image_list(
-            images=type_filtered_images,
-            type_filters=None,
-            attribute_filters=job_attribute_filters,
-        )
+        if wftask.task_type in ["compound", "parallel", "non_parallel"]:
+            type_filters = copy(current_dataset_type_filters)
+            type_filters_patch = merge_type_filters(
+                task_input_types=task.input_types,
+                wftask_type_filters=wftask.type_filters,
+            )
+            type_filters.update(type_filters_patch)
+            type_filtered_images = filter_image_list(
+                images=tmp_images,
+                type_filters=type_filters,
+                attribute_filters=None,
+            )
+            num_available_images = len(type_filtered_images)
+            filtered_images = filter_image_list(
+                images=type_filtered_images,
+                type_filters=None,
+                attribute_filters=job_attribute_filters,
+            )
+        else:
+            num_available_images = 0
 
         # Create history item
         with next(get_sync_db()) as db:
@@ -100,7 +106,7 @@ def execute_tasks_v2(
                 workflowtask_id=wftask.id,
                 workflowtask_dump=workflowtask_dump,
                 task_group_dump=task_group_dump,
-                num_available_images=len(type_filtered_images),
+                num_available_images=num_available_images,
                 status=XXXStatus.SUBMITTED,
             )
             db.add(history_run)
@@ -126,6 +132,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_non_parallel":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_non_parallel(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         elif task.type == "parallel":
             current_task_output, num_tasks, exceptions = run_v2_task_parallel(
                 images=filtered_images,
@@ -151,6 +173,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_compound":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_compound(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         else:
             raise ValueError(f"Unexpected error: Invalid {task.type=}.")
 

fractal_server/app/runner/v2/runner_functions.py

@@ -155,6 +155,92 @@ def run_v2_task_non_parallel(
             root_dir_remote=workflow_dir_remote,
         ),
         parameters=function_kwargs,
+        task_type="non_parallel",
+        **executor_options,
+    )
+
+    num_tasks = 1
+    with next(get_sync_db()) as db:
+        if exception is None:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.DONE)
+            )
+            db.commit()
+            if result is None:
+                return (TaskOutput(), num_tasks, {})
+            else:
+                return (_cast_and_validate_TaskOutput(result), num_tasks, {})
+        else:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.FAILED)
+            )
+            db.commit()
+            return (TaskOutput(), num_tasks, {0: exception})
+
+
+def run_v2_task_converter_non_parallel(
+    *,
+    zarr_dir: str,
+    task: TaskV2,
+    wftask: WorkflowTaskV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    executor: BaseRunner,
+    submit_setup_call: callable = no_op_submit_setup_call,
+    dataset_id: int,
+    history_run_id: int,
+) -> tuple[TaskOutput, int, dict[int, BaseException]]:
+    """
+    This runs server-side (see `executor` argument)
+    """
+
+    if workflow_dir_remote is None:
+        workflow_dir_remote = workflow_dir_local
+        logging.warning(
+            "In `run_single_task`, workflow_dir_remote=None. Is this right?"
+        )
+        workflow_dir_remote = workflow_dir_local
+
+    executor_options = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="non_parallel",
+    )
+
+    function_kwargs = {
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: _index_to_component(0),
+        **(wftask.args_non_parallel or {}),
+    }
+
+    # Database History operations
+    with next(get_sync_db()) as db:
+        history_unit = HistoryUnit(
+            history_run_id=history_run_id,
+            status=XXXStatus.SUBMITTED,
+            logfile=None,  # FIXME
+            zarr_urls=[],
+        )
+        db.add(history_unit)
+        db.commit()
+        db.refresh(history_unit)
+        history_unit_id = history_unit.id
+
+    result, exception = executor.submit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_non_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        task_type="converter_non_parallel",
+        parameters=function_kwargs,
         **executor_options,
     )
 
@@ -255,6 +341,7 @@ def run_v2_task_parallel(
             root_dir_remote=workflow_dir_remote,
         ),
         list_parameters=list_function_kwargs,
+        task_type="parallel",
         **executor_options,
     )
 
@@ -365,6 +452,159 @@ def run_v2_task_compound(
             root_dir_remote=workflow_dir_remote,
         ),
         parameters=function_kwargs,
+        task_type="compound",
+        **executor_options_init,
+    )
+
+    num_tasks = 1
+    if exception is None:
+        if result is None:
+            init_task_output = InitTaskOutput()
+        else:
+            init_task_output = _cast_and_validate_InitTaskOutput(result)
+    else:
+        with next(get_sync_db()) as db:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.FAILED)
+            )
+            db.commit()
+        return (TaskOutput(), num_tasks, {0: exception})
+
+    parallelization_list = init_task_output.parallelization_list
+    parallelization_list = deduplicate_list(parallelization_list)
+
+    num_tasks = 1 + len(parallelization_list)
+
+    # 3/B: parallel part of a compound task
+    _check_parallelization_list_size(parallelization_list)
+
+    if len(parallelization_list) == 0:
+        with next(get_sync_db()) as db:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.DONE)
+            )
+            db.commit()
+        return (TaskOutput(), 0, {})
+
+    list_function_kwargs = [
+        {
+            "zarr_url": parallelization_item.zarr_url,
+            "init_args": parallelization_item.init_args,
+            _COMPONENT_KEY_: f"compute_{_index_to_component(ind)}",
+            **(wftask.args_parallel or {}),
+        }
+        for ind, parallelization_item in enumerate(parallelization_list)
+    ]
+
+    results, exceptions = executor.multisubmit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        list_parameters=list_function_kwargs,
+        task_type="compound",
+        **executor_options_compute,
+    )
+
+    outputs = []
+    failure = False
+    for ind in range(len(list_function_kwargs)):
+        if ind in results.keys():
+            result = results[ind]
+            if result is None:
+                output = TaskOutput()
+            else:
+                output = _cast_and_validate_TaskOutput(result)
+            outputs.append(output)
+
+        elif ind in exceptions.keys():
+            print(f"Bad: {exceptions[ind]}")
+            failure = True
+        else:
+            print("VERY BAD - should have not reached this point")
+
+    with next(get_sync_db()) as db:
+        if failure:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.FAILED)
+            )
+        else:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=XXXStatus.DONE)
+            )
+        db.commit()
+
+    merged_output = merge_outputs(outputs)
+    return (merged_output, num_tasks, exceptions)
+
+
+def run_v2_task_converter_compound(
+    *,
+    zarr_dir: str,
+    task: TaskV2,
+    wftask: WorkflowTaskV2,
+    executor: BaseRunner,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    submit_setup_call: callable = no_op_submit_setup_call,
+    dataset_id: int,
+    history_run_id: int,
+) -> tuple[TaskOutput, int, dict[int, BaseException]]:
+    executor_options_init = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="non_parallel",
+    )
+    executor_options_compute = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="parallel",
+    )
+
+    # 3/A: non-parallel init task
+    function_kwargs = {
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: f"init_{_index_to_component(0)}",
+        **(wftask.args_non_parallel or {}),
+    }
+
+    # Create database History entries
+    with next(get_sync_db()) as db:
+        # Create a single `HistoryUnit` for the whole compound task
+        history_unit = HistoryUnit(
+            history_run_id=history_run_id,
+            status=XXXStatus.SUBMITTED,
+            logfile=None,  # FIXME
+            zarr_urls=[],
+        )
+        db.add(history_unit)
+        db.commit()
+        db.refresh(history_unit)
+        history_unit_id = history_unit.id
+
+    result, exception = executor.submit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_non_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        parameters=function_kwargs,
+        task_type="converter_compound",
         **executor_options_init,
     )
 
@@ -421,7 +661,7 @@ def run_v2_task_compound(
             root_dir_remote=workflow_dir_remote,
         ),
         list_parameters=list_function_kwargs,
-        in_compound_task=True,
+        task_type="converter_compound",
         **executor_options_compute,
     )
 

fractal_server/app/schemas/v2/manifest.py

@@ -1,4 +1,5 @@
 from typing import Any
+from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
@@ -56,6 +57,16 @@ class TaskManifestV2(BaseModel):
     modality: Optional[str] = None
     tags: list[str] = Field(default_factory=list)
 
+    type: Optional[
+        Literal[
+            "compound",
+            "converter_compound",
+            "non_parallel",
+            "converter_non_parallel",
+            "parallel",
+        ]
+    ] = None
+
     @model_validator(mode="after")
     def validate_executable_args_meta(self):
         executable_non_parallel = self.executable_non_parallel

fractal_server/app/schemas/v2/task.py

@@ -13,8 +13,20 @@ from .._validators import cant_set_none
 from fractal_server.app.schemas._validators import NonEmptyString
 from fractal_server.app.schemas._validators import val_unique_list
 from fractal_server.app.schemas._validators import valdict_keys
+from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
 
+TaskTypeType = Literal[
+    "compound",
+    "converter_compound",
+    "non_parallel",
+    "converter_non_parallel",
+    "parallel",
+]
+
+
+logger = set_logger(__name__)
+
 
 class TaskCreateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
@@ -41,6 +53,8 @@ class TaskCreateV2(BaseModel):
     tags: list[NonEmptyString] = Field(default_factory=list)
     authors: Optional[NonEmptyString] = None
 
+    type: Optional[TaskTypeType] = None
+
     # Validators
 
     @field_validator(
@@ -69,6 +83,23 @@ class TaskCreateV2(BaseModel):
 
         return self
 
+    @model_validator(mode="after")
+    def set_task_type(self):
+        if self.type is None:
+            logger.warning(
+                f"Task type is not set for task '{self.name}', "
+                "which will be deprecated in a future version. "
+                "Please move to `fractal-task-tools`."
+            )
+            if self.command_non_parallel is None:
+                self.type = "parallel"
+            elif self.command_parallel is None:
+                self.type = "non_parallel"
+            else:
+                self.type = "compound"
+
+        return self
+
     _meta_non_parallel = field_validator("meta_non_parallel")(
         classmethod(valdict_keys("meta_non_parallel"))
     )
@@ -127,6 +158,8 @@ class TaskReadV2(BaseModel):
     authors: Optional[str] = None
     tags: list[str]
 
+    type: Optional[TaskTypeType] = None
+
 
 class TaskUpdateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")

fractal_server/app/schemas/v2/workflowtask.py

@@ -16,6 +16,7 @@ from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
+from .task import TaskTypeType
 
 RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
 
@@ -113,7 +114,7 @@ class WorkflowTaskReadV2(BaseModel):
 
     type_filters: dict[str, bool]
 
-    task_type: str
+    task_type: TaskTypeType
     task_id: int
     task: TaskReadV2
 

fractal_server/tasks/v2/utils_database.py

@@ -5,15 +5,6 @@ from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.schemas.v2 import TaskCreateV2
 
 
-def _get_task_type(task: TaskCreateV2) -> str:
-    if task.command_non_parallel is None:
-        return "parallel"
-    elif task.command_parallel is None:
-        return "non_parallel"
-    else:
-        return "compound"
-
-
 def create_db_tasks_and_update_task_group(
     *,
     task_group_id: int,
@@ -31,13 +22,7 @@ def create_db_tasks_and_update_task_group(
     Returns:
         Updated `TaskGroupV2` object.
     """
-    actual_task_list = [
-        TaskV2(
-            **task.model_dump(),
-            type=_get_task_type(task),
-        )
-        for task in task_list
-    ]
+    actual_task_list = [TaskV2(**task.model_dump()) for task in task_list]
     task_group = db.get(TaskGroupV2, task_group_id)
     task_group.task_list = actual_task_list
     db.add(task_group)

{fractal_server-2.14.0a6.dist-info → fractal_server-2.14.0a7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a6
+Version: 2.14.0a7
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.14.0a6.dist-info → fractal_server-2.14.0a7.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=AnugSNpe3C8nki8yFEbYV-Vn26Um07z3ucouM8Qt1BE,25
+fractal_server/__init__.py,sha256=1JU9zoVeJhUgQtGc2wlEGMoG-HAWT8I6UZKOOCDkSGA,25
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -44,14 +44,14 @@ fractal_server/app/routes/api/v2/job.py,sha256=m89FTh9Px25oXCeWj2k2NdGWQaO2oxMh-
 fractal_server/app/routes/api/v2/project.py,sha256=hMvL9QLPUcAAiPGy6ta2LBLTVRozJsfvBPl5D06_MHg,6666
 fractal_server/app/routes/api/v2/status_legacy.py,sha256=sJLHGGHI9so_Sa4-8JuhMGBPeE6n4K2DmDuiw6IB4XY,6317
 fractal_server/app/routes/api/v2/submit.py,sha256=K4OjcSg476JXIeeMUaYdTDk8Qpj5IO5UULvfErI7Y5Y,8624
-fractal_server/app/routes/api/v2/task.py,sha256=z3_SxsXoKsbM9GGNJUdIiZisQwAJSBqvCc7thaJIOTU,7191
+fractal_server/app/routes/api/v2/task.py,sha256=O7pquZhXIS4lRs5XqHvstiwe8BiCuS-B3ZKJI1g6EJU,6985
 fractal_server/app/routes/api/v2/task_collection.py,sha256=IDNF6sjDuU37HIQ0TuQA-TZIuf7nfHAQXUUNmkrlhLM,12706
 fractal_server/app/routes/api/v2/task_collection_custom.py,sha256=cctW61-C2QYF2KXluS15lLhZJS_kt30Ca6UGLFO32z0,6207
 fractal_server/app/routes/api/v2/task_group.py,sha256=j3zDvVZizB7NWEgVgZU42JCXETkaVkk2ImJPr0jS7BQ,8164
 fractal_server/app/routes/api/v2/task_group_lifecycle.py,sha256=3o9bCC8ubMwffQPPaxQZy-CjH9IB2RkIReIecI6L2_w,9300
 fractal_server/app/routes/api/v2/workflow.py,sha256=U3iZX5IiFAJ20-R8IjlYToOdm9gXsmtr1lW7ASEH9P8,11689
 fractal_server/app/routes/api/v2/workflow_import.py,sha256=INmnhlMEBJp-vHPR0f940DANPmIidts3OfcooeM_aNA,11205
-fractal_server/app/routes/api/v2/workflowtask.py,sha256=pi4oxWszNgWkDm6oQavanmq4_P5kpcgN3Dfjz9XAuV8,11063
+fractal_server/app/routes/api/v2/workflowtask.py,sha256=7_syX2EO7ibF6Xkm7HBPhsUYq6aYnKNeC5iSaafQhG4,11342
 fractal_server/app/routes/auth/__init__.py,sha256=fao6CS0WiAjHDTvBzgBVV_bSXFpEAeDBF6Z6q7rRkPc,1658
 fractal_server/app/routes/auth/_aux_auth.py,sha256=UZgauY0V6mSqjte_sYI1cBl2h8bcbLaeWzgpl1jdJlk,4883
 fractal_server/app/routes/auth/current_user.py,sha256=FUegTahlxT3BdPVCQYir0-ogg2YAaZ1DYuLcE_5NC9Y,5906
@@ -71,11 +71,11 @@ fractal_server/app/runner/components.py,sha256=ZF8ct_Ky5k8IAcrmpYOZ-bc6OBgdELEig
 fractal_server/app/runner/compress_folder.py,sha256=HSc1tv7x2DBjBoXwugZlC79rm9GNBIWtQKK9yWn5ZBI,3991
 fractal_server/app/runner/exceptions.py,sha256=_qZ_t8O4umAdJ1ikockiF5rDJuxnEskrGrLjZcnQl7A,4159
 fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fractal_server/app/runner/executors/base_runner.py,sha256=0E3gbSndXdEAxZwFCiZXrUd8tjEmvLa_ztPBGMJXtUw,3742
+fractal_server/app/runner/executors/base_runner.py,sha256=v-uUHEvJgRnipTEvU9AmIMkYbkkazqkjm4iAF1GXHEM,4562
 fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/local/_local_config.py,sha256=8dyg2Gh8L2FlG_jJRYLMkcMgVHGEY2w7DME9aaKXFFo,3688
 fractal_server/app/runner/executors/local/_submit_setup.py,sha256=pDc9Q6axXL8_5JAV0byXzGOLOB0bZF88_L9LZykOgwM,1220
-fractal_server/app/runner/executors/local/runner.py,sha256=Iy5Pc8rdkj7IGcwUbkO_UgZNDytXtfMysYtMSu_dlY0,5498
+fractal_server/app/runner/executors/local/runner.py,sha256=LNql8q6M-Cn_hEV4IMkNP57XFPQJ6eaVd0YIDKJLk60,5621
 fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
@@ -91,7 +91,7 @@ fractal_server/app/runner/executors/slurm_ssh/executor.py,sha256=JW6zguEy9XsHebS
 fractal_server/app/runner/executors/slurm_sudo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_sudo/_check_jobs_status.py,sha256=eZd9lxbObsqc1M3B96IGMJ-1oC0jo8lBOX4Nto97VvE,2036
 fractal_server/app/runner/executors/slurm_sudo/_subprocess_run_as_user.py,sha256=O1bNg1DiSDJmQE0RmOk2Ii47DagiXp5ryd0R6KxO2OM,3177
-fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=VOWd8bYzCU8C6TziZN7VUurX52lbvNDWLZ2Ht3uvnGc,24057
+fractal_server/app/runner/executors/slurm_sudo/runner.py,sha256=niDcezFNQ7awfv_7nqABmVhG3oLIf8Z7BwlLxM-chZo,21833
 fractal_server/app/runner/extract_archive.py,sha256=tLpjDrX47OjTNhhoWvm6iNukg8KoieWyTb7ZfvE9eWU,2483
 fractal_server/app/runner/filenames.py,sha256=lPnxKHtdRizr6FqG3zOdjDPyWA7GoaJGTtiuJV0gA8E,70
 fractal_server/app/runner/run_subprocess.py,sha256=c3JbYXq3hX2aaflQU19qJ5Xs6J6oXGNvnTEoAfv2bxc,959
@@ -105,8 +105,8 @@ fractal_server/app/runner/v2/_slurm_ssh.py,sha256=5w_lwQzySx-R3kVg2Bf-21n5JpWjJA
 fractal_server/app/runner/v2/_slurm_sudo.py,sha256=CzWUeC6at_Sj-wU1myjA68ZRKMiLZYBTLv9I9odUxBU,2914
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
-fractal_server/app/runner/v2/runner.py,sha256=25kOKm4B7_BhUlg7Dh0yzIcJ2izugA_WFNhVf06O0Y4,13970
-fractal_server/app/runner/v2/runner_functions.py,sha256=aH3N_7U_-pqiykvieEVk-W9Y0ir2YohJ9DY8As9zUc8,14445
+fractal_server/app/runner/v2/runner.py,sha256=3M6xGeccY_AZKua305MgWLuCuhkACoqwKgTV4vuNguk,15534
+fractal_server/app/runner/v2/runner_functions.py,sha256=tztuEF7m0iE-ca5n6Vb-l1dDUldQXAGxZkD1OTL6I6w,21889
 fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=dvvRK7od8iQ8vdPf80uGUxs3i5i0buGjCodBxSjZ7PQ,3671
 fractal_server/app/runner/v2/task_interface.py,sha256=e1GGQSYd0MyBj1EZvEVzqv-HpVE4YffXOq82WLrCaOc,1866
 fractal_server/app/runner/versions.py,sha256=dSaPRWqmFPHjg20kTCHmi_dmGNcCETflDtDLronNanU,852
@@ -121,14 +121,14 @@ fractal_server/app/schemas/v2/accounting.py,sha256=Wylt7uWTiDIFlHJOh4XEtYitk2FjF
 fractal_server/app/schemas/v2/dataset.py,sha256=9yc-tte70yPPk4CSfy2imykYVbCW8-23K499pi9z2e0,5206
 fractal_server/app/schemas/v2/dumps.py,sha256=2GUjoqeblUvrSoojBz5odoUUf53IABtbY_5GvFZoMVc,1782
 fractal_server/app/schemas/v2/job.py,sha256=KhxQOfncpE_SAu7Wed8CXS2G6onh0v875GkotBvKBTY,4304
-fractal_server/app/schemas/v2/manifest.py,sha256=tMNKtwVUBhVyAmBHK6f2agzvdKuYhuTP-e2cG9t66y0,7045
+fractal_server/app/schemas/v2/manifest.py,sha256=8mmB0QwxEgAeGgwKD_fT-o-wFy7lb6HxNXbp17IJqNY,7281
 fractal_server/app/schemas/v2/project.py,sha256=ulgCmUnX0w-0jrSjVYIT7sxeK95CSNGh2msXydhsgYI,885
 fractal_server/app/schemas/v2/status.py,sha256=SQaUpQkjFq5c5k5J4rOjNhuQaDOEg8lksPhkKmPU5VU,332
-fractal_server/app/schemas/v2/task.py,sha256=lPXxhwvPLjXuoZGdVSLZq3wTChfBbP7V-uTEpQa1HoU,5753
+fractal_server/app/schemas/v2/task.py,sha256=-oBE5-2tSUd6u75duAcaFyEFGk3MsQzrdBr_mFldOBc,6627
 fractal_server/app/schemas/v2/task_collection.py,sha256=dLu4sy-su5k5vDJqCZdJMW8mLT5TX2SV60l_RAvKhwY,5930
 fractal_server/app/schemas/v2/task_group.py,sha256=A3SFHNHLKPJyrnDz-wbnQvycetafKltp6UsH1u-euwA,3850
 fractal_server/app/schemas/v2/workflow.py,sha256=ZpM43zTMyLRnEUtkbr_J5DYP00NwjItaC8gweB7GGAA,2172
-fractal_server/app/schemas/v2/workflowtask.py,sha256=qMvwlnFCsnyD8uv8HJ4cFy2-QMm2ETUFlTIbxIFUWxk,8056
+fractal_server/app/schemas/v2/workflowtask.py,sha256=qOzpsWQ7QUNey-rK9hPB3PKFrRxXrMvcRzqZxRuEYAI,8096
 fractal_server/app/security/__init__.py,sha256=e2cveg5hQpieGD3bSPd5GTOMthvJ-HXH3buSb9WVfEU,14096
 fractal_server/app/security/signup_email.py,sha256=Xd6QYxcdmg0PHpDwmUE8XQmPcOj3Xjy5oROcIMhmltM,1472
 fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
@@ -199,15 +199,15 @@ fractal_server/tasks/v2/templates/4_pip_show.sh,sha256=qm1vPy6AkKhWDjCJGXS8LqCLY
 fractal_server/tasks/v2/templates/5_get_venv_size_and_file_number.sh,sha256=q-6ZUvA6w6FDVEoSd9O63LaJ9tKZc7qAFH72SGPrd_k,284
 fractal_server/tasks/v2/templates/6_pip_install_from_freeze.sh,sha256=A2y8RngEjAcRhG-_owA6P7tAdrS_AszFuGXnaeMV8u0,1122
 fractal_server/tasks/v2/utils_background.py,sha256=W_RvihI1aiYPJNsPo8z4wKuA_bPs0UT2huzLihRpjU4,4248
-fractal_server/tasks/v2/utils_database.py,sha256=iLbwkxMxTCgpyKe1JQzdfIR3zBfxohgmLwSdGps1AUo,1274
+fractal_server/tasks/v2/utils_database.py,sha256=h4Pa-JxcVk7WA4_Pz8CxFT9sX3sA43p2rVRg7FVSgvY,967
 fractal_server/tasks/v2/utils_package_names.py,sha256=RDg__xrvQs4ieeVzmVdMcEh95vGQYrv9Hfal-5EDBM8,2393
 fractal_server/tasks/v2/utils_python_interpreter.py,sha256=5_wrlrTqXyo1YuLZvAW9hrSoh5MyLOzdPVUlUwM7uDQ,955
 fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENPqJeTQEFiz4bOg,3124
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a6.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a6.dist-info/METADATA,sha256=uuUN7vjLvxCYZIqs3uvcubWLPb_PLnaQNrtg4U1DHck,4550
-fractal_server-2.14.0a6.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fractal_server-2.14.0a6.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a6.dist-info/RECORD,,
+fractal_server-2.14.0a7.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a7.dist-info/METADATA,sha256=-H-lBPwP4S9jTTot-k3weBihbHTA2utPTxzC1z7h8so,4550
+fractal_server-2.14.0a7.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fractal_server-2.14.0a7.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a7.dist-info/RECORD,,