fractal-server 2.14.0a6__py3-none-any.whl → 2.14.0a8__py3-none-any.whl

fractal_server/app/runner/executors/base_runner.py

@@ -1,6 +1,20 @@
 from typing import Any
 
 from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.schemas.v2.task import TaskTypeType
+
+
+TASK_TYPES_SUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "non_parallel",
+    "converter_non_parallel",
+]
+TASK_TYPES_MULTISUBMIT: list[TaskTypeType] = [
+    "compound",
+    "converter_compound",
+    "parallel",
+]
 
 
 class BaseRunner(object):
@@ -16,7 +30,7 @@ class BaseRunner(object):
         func: callable,
         parameters: dict[str, Any],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[Any, BaseException]:
         """
@@ -25,16 +39,13 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             parameters:
                 Dictionary of parameters. Must include `zarr_urls` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the init part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
@@ -43,7 +54,7 @@ class BaseRunner(object):
         func: callable,
         list_parameters: list[dict[str, Any]],
         history_item_id: int,
-        in_compound_task: bool,
+        task_type: TaskTypeType,
         **kwargs,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
@@ -52,33 +63,44 @@ class BaseRunner(object):
         # FIXME: Describe more in detail
 
         Args:
-            func:
-                Function to be executed.
+            func: Function to be executed.
             list_parameters:
                 List of dictionaries of parameters. Each one must include a
                 `zarr_url` key.
             history_item_id:
                 Database ID of the corresponding `HistoryItemV2` entry.
-            in_compound_task:
-                Whether this is the compute part of a compound task.
-            kwargs:
-                Runner-specific parameters.
+            task_type: Task type.
+            kwargs: Runner-specific parameters.
         """
         raise NotImplementedError()
 
-    def validate_submit_parameters(self, parameters: dict[str, Any]) -> None:
+    def validate_submit_parameters(
+        self,
+        parameters: dict[str, Any],
+        task_type: TaskTypeType,
+    ) -> None:
         """
         Validate parameters for `submit` method
 
         Args:
             parameters: Parameters dictionary.
+            task_type: Task type.s
         """
+        if task_type not in TASK_TYPES_SUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `submit`.")
         if not isinstance(parameters, dict):
             raise ValueError("`parameters` must be a dictionary.")
-        if "zarr_urls" not in parameters.keys():
-            raise ValueError(
-                f"No 'zarr_urls' key in in {list(parameters.keys())}"
-            )
+        if task_type in ["non_parallel", "compound"]:
+            if "zarr_urls" not in parameters.keys():
+                raise ValueError(
+                    f"No 'zarr_urls' key in in {list(parameters.keys())}"
+                )
+        elif task_type in ["converter_non_parallel", "converter_compound"]:
+            if "zarr_urls" in parameters.keys():
+                raise ValueError(
+                    f"Forbidden 'zarr_urls' key in {list(parameters.keys())}"
+                )
+
         if _COMPONENT_KEY_ not in parameters.keys():
             raise ValueError(
                 f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
@@ -87,21 +109,26 @@ class BaseRunner(object):
     def validate_multisubmit_parameters(
         self,
         list_parameters: list[dict[str, Any]],
-        in_compound_task: bool,
+        task_type: TaskTypeType,
     ) -> None:
         """
         Validate parameters for `multi_submit` method
 
         Args:
             list_parameters: List of parameters dictionaries.
-            in_compound_task:
-               Whether this is the compute part of a compound task.
+            task_type: Task type.
         """
+        if task_type not in TASK_TYPES_MULTISUBMIT:
+            raise ValueError(f"Invalid {task_type=} for `multisubmit`.")
+
+        if not isinstance(list_parameters, list):
+            raise ValueError("`parameters` must be a list.")
+
         for single_kwargs in list_parameters:
             if not isinstance(single_kwargs, dict):
-                raise RuntimeError("kwargs itemt must be a dictionary.")
+                raise ValueError("kwargs itemt must be a dictionary.")
             if "zarr_url" not in single_kwargs.keys():
-                raise RuntimeError(
+                raise ValueError(
                     f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
                 )
             if _COMPONENT_KEY_ not in single_kwargs.keys():
@@ -109,7 +136,7 @@ class BaseRunner(object):
                     f"No '{_COMPONENT_KEY_}' key "
                     f"in {list(single_kwargs.keys())}"
                 )
-        if not in_compound_task:
+        if task_type == "parallel":
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):
-                raise RuntimeError("Non-unique zarr_urls")
+                raise ValueError("Non-unique zarr_urls")

fractal_server/app/runner/executors/local/runner.py

@@ -9,9 +9,9 @@ from ._local_config import LocalBackendConfig
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.logger import set_logger
 
-
 logger = set_logger(__name__)
 
 
@@ -50,6 +50,7 @@ class LocalRunner(BaseRunner):
         func: callable,
         parameters: dict[str, Any],
         task_files: TaskFiles,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
@@ -61,7 +62,7 @@ class LocalRunner(BaseRunner):
             component=parameters[_COMPONENT_KEY_],
         )
 
-        self.validate_submit_parameters(parameters)
+        self.validate_submit_parameters(parameters, task_type=task_type)
         workdir_local = current_task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
@@ -83,18 +84,18 @@ class LocalRunner(BaseRunner):
         func: callable,
         list_parameters: list[dict],
         task_files: TaskFiles,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
         local_backend_config: Optional[LocalBackendConfig] = None,
     ):
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            task_type=task_type,
         )
 
         workdir_local = task_files.wftask_subfolder_local
-        if not in_compound_task:
+        if task_type not in ["compound", "converter_compound"]:
             workdir_local.mkdir()
 
         # Get local_backend_config

fractal_server/app/runner/executors/slurm_ssh/executor.py

@@ -9,7 +9,6 @@ from concurrent.futures import InvalidStateError
 from copy import copy
 from pathlib import Path
 from typing import Any
-from typing import Callable
 from typing import Optional
 from typing import Sequence
 
@@ -216,7 +215,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def submit(
         self,
-        fun: Callable[..., Any],
+        fun: callable,
         *fun_args: Sequence[Any],
         slurm_config: SlurmConfig,
         task_files: TaskFiles,
@@ -278,7 +277,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def map(
         self,
-        fn: Callable[..., Any],
+        fn: callable,
         iterable: list[Sequence[Any]],
         *,
         slurm_config: SlurmConfig,
@@ -446,7 +445,7 @@ class FractalSlurmSSHExecutor(Executor):
 
     def _prepare_job(
         self,
-        fun: Callable[..., Any],
+        fun: callable,
         slurm_file_prefix: str,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,

fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -30,15 +30,11 @@ from fractal_server.app.runner.executors.slurm_common._slurm_config import (
 )
 from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
-# from fractal_server.app.history import ImageStatus
-# from fractal_server.app.history import update_all_images
-# from fractal_server.app.history import update_single_image
-# from fractal_server.app.history import update_single_image_logfile
-
 
 logger = set_logger(__name__)
 
@@ -426,7 +422,7 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ) -> tuple[Any, Exception]:
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
@@ -439,21 +435,9 @@ class RunnerSlurmSudo(BaseRunner):
         )
 
         if self.jobs != {}:
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Unexpected branch: jobs should be empty.")
 
         if self.is_shutdown():
-            if not in_compound_task:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                # )
             raise JobExecutionError("Cannot continue after shutdown.")
 
         # Validation phase
@@ -505,22 +489,6 @@ class RunnerSlurmSudo(BaseRunner):
                 )
             time.sleep(self.slurm_poll_interval)
 
-        if not in_compound_task:
-            if exception is None:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.DONE,
-                #     logfile=LOGFILE,
-                # )
-            else:
-                pass
-                # update_all_images(
-                #     history_item_id=history_item_id,
-                #     status=ImageStatus.FAILED,
-                #     logfile=LOGFILE,
-                # )
-
         return result, exception
 
     def multisubmit(
@@ -530,20 +498,19 @@ class RunnerSlurmSudo(BaseRunner):
         history_item_id: int,
         task_files: TaskFiles,
         slurm_config: SlurmConfig,
-        in_compound_task: bool = False,
+        task_type: TaskTypeType,
     ):
         # self.scancel_jobs()
 
         self.validate_multisubmit_parameters(
-            list_parameters=list_parameters,
-            in_compound_task=in_compound_task,
+            list_parameters=list_parameters, task_type=task_type
         )
 
         workdir_local = task_files.wftask_subfolder_local
         workdir_remote = task_files.wftask_subfolder_remote
 
         # Create local&remote task subfolders
-        if not in_compound_task:
+        if task_type not in ["converter_compound", "compound"]:
             original_umask = os.umask(0)
             workdir_local.mkdir(parents=True, mode=0o755)
             os.umask(original_umask)
@@ -640,28 +607,6 @@ class RunnerSlurmSudo(BaseRunner):
                     result, exception = self._postprocess_single_task(
                         task=task
                     )
-                    if not in_compound_task:
-                        pass
-                        # update_single_image_logfile(
-                        #     history_item_id=history_item_id,
-                        #     zarr_url=task.zarr_url,
-                        #     logfile=task.task_files.log_file_local,
-                        # )
-                    if not in_compound_task:
-                        if exception is None:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.DONE,
-                            # )
-                        else:
-                            pass
-                            # update_single_image(
-                            #     zarr_url=task.zarr_url,
-                            #     history_item_id=history_item_id,
-                            #     status=ImageStatus.FAILED,
-                            # )
                     if exception is None:
                         results[task.index] = result
                     else:

fractal_server/app/runner/v2/runner.py

@@ -2,7 +2,6 @@ import logging
 from copy import copy
 from copy import deepcopy
 from pathlib import Path
-from typing import Callable
 from typing import Optional
 
 from sqlalchemy.orm.attributes import flag_modified
@@ -14,17 +13,21 @@ from ....images.tools import find_image_by_zarr_url
 from ..exceptions import JobExecutionError
 from .runner_functions import no_op_submit_setup_call
 from .runner_functions import run_v2_task_compound
+from .runner_functions import run_v2_task_converter_compound
+from .runner_functions import run_v2_task_converter_non_parallel
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import TaskDumpV2
+from fractal_server.app.schemas.v2 import TaskGroupDumpV2
 from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import merge_type_filters
 
@@ -38,7 +41,7 @@ def execute_tasks_v2(
     workflow_dir_local: Path,
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
-    submit_setup_call: Callable = no_op_submit_setup_call,
+    submit_setup_call: callable = no_op_submit_setup_call,
     job_attribute_filters: AttributeFiltersType,
 ) -> None:
     logger = logging.getLogger(logger_name)
@@ -63,45 +66,45 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Filter images by types and attributes (in two steps)
-        type_filters = copy(current_dataset_type_filters)
-        type_filters_patch = merge_type_filters(
-            task_input_types=task.input_types,
-            wftask_type_filters=wftask.type_filters,
-        )
-        type_filters.update(type_filters_patch)
-        type_filtered_images = filter_image_list(
-            images=tmp_images,
-            type_filters=type_filters,
-            attribute_filters=None,
-        )
-        filtered_images = filter_image_list(
-            images=type_filtered_images,
-            type_filters=None,
-            attribute_filters=job_attribute_filters,
-        )
+        if wftask.task_type in ["compound", "parallel", "non_parallel"]:
+            type_filters = copy(current_dataset_type_filters)
+            type_filters_patch = merge_type_filters(
+                task_input_types=task.input_types,
+                wftask_type_filters=wftask.type_filters,
+            )
+            type_filters.update(type_filters_patch)
+            type_filtered_images = filter_image_list(
+                images=tmp_images,
+                type_filters=type_filters,
+                attribute_filters=None,
+            )
+            num_available_images = len(type_filtered_images)
+            filtered_images = filter_image_list(
+                images=type_filtered_images,
+                type_filters=None,
+                attribute_filters=job_attribute_filters,
+            )
+        else:
+            num_available_images = 0
 
-        # Create history item
         with next(get_sync_db()) as db:
+            # Create dumps for workflowtask and taskgroup
             workflowtask_dump = dict(
                 **wftask.model_dump(exclude={"task"}),
-                task=wftask.task.model_dump(),
+                task=TaskDumpV2(**wftask.task.model_dump()).model_dump(),
             )
-
-            # Exclude timestamps since they'd need to be serialized properly
             task_group = db.get(TaskGroupV2, wftask.task.taskgroupv2_id)
-            task_group_dump = task_group.model_dump(
-                exclude={
-                    "timestamp_created",
-                    "timestamp_last_used",
-                }
-            )
+            task_group_dump = TaskGroupDumpV2(
+                **task_group.model_dump()
+            ).model_dump()
+            # Create HistoryRun
             history_run = HistoryRun(
                 dataset_id=dataset.id,
                 workflowtask_id=wftask.id,
                 workflowtask_dump=workflowtask_dump,
                 task_group_dump=task_group_dump,
-                num_available_images=len(type_filtered_images),
-                status=XXXStatus.SUBMITTED,
+                num_available_images=num_available_images,
+                status=HistoryUnitStatus.SUBMITTED,
             )
             db.add(history_run)
             db.commit()
@@ -126,6 +129,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_non_parallel":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_non_parallel(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         elif task.type == "parallel":
             current_task_output, num_tasks, exceptions = run_v2_task_parallel(
                 images=filtered_images,
@@ -151,6 +170,22 @@ def execute_tasks_v2(
                 history_run_id=history_run_id,
                 dataset_id=dataset.id,
             )
+        elif task.type == "converter_compound":
+            (
+                current_task_output,
+                num_tasks,
+                exceptions,
+            ) = run_v2_task_converter_compound(
+                zarr_dir=zarr_dir,
+                wftask=wftask,
+                task=task,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
+                executor=runner,
+                submit_setup_call=submit_setup_call,
+                history_run_id=history_run_id,
+                dataset_id=dataset.id,
+            )
         else:
             raise ValueError(f"Unexpected error: Invalid {task.type=}.")
 
@@ -323,14 +358,14 @@ def execute_tasks_v2(
                 db.execute(
                     update(HistoryRun)
                     .where(HistoryRun.id == history_run_id)
-                    .values(status=XXXStatus.DONE)
+                    .values(status=HistoryUnitStatus.DONE)
                 )
                 db.commit()
             else:
                 db.execute(
                     update(HistoryRun)
                     .where(HistoryRun.id == history_run_id)
-                    .values(status=XXXStatus.FAILED)
+                    .values(status=HistoryUnitStatus.FAILED)
                 )
                 db.commit()
                 logger.error(