fractal-server 2.14.12__py3-none-any.whl → 2.14.14__py3-none-any.whl

fractal_server/app/runner/executors/local/runner.py

@@ -3,19 +3,21 @@ from concurrent.futures import Future
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Any
-from typing import Literal
 
 from ..call_command_wrapper import call_command_wrapper
 from .get_local_config import LocalBackendConfig
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.runner.executors.base_runner import MultisubmitTaskType
+from fractal_server.app.runner.executors.base_runner import SubmitTaskType
 from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.app.runner.v2.db_tools import (
     bulk_update_status_of_history_unit,
 )
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import TaskType
 from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
@@ -87,12 +89,7 @@ class LocalRunner(BaseRunner):
         history_unit_id: int,
         task_files: TaskFiles,
         config: LocalBackendConfig,
-        task_type: Literal[
-            "non_parallel",
-            "converter_non_parallel",
-            "compound",
-            "converter_compound",
-        ],
+        task_type: SubmitTaskType,
         user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
@@ -129,7 +126,10 @@ class LocalRunner(BaseRunner):
             try:
                 result = future.result()
                 logger.debug("[submit] END with result")
-                if task_type not in ["compound", "converter_compound"]:
+                if task_type not in [
+                    TaskType.COMPOUND,
+                    TaskType.CONVERTER_COMPOUND,
+                ]:
                     update_status_of_history_unit(
                         history_unit_id=history_unit_id,
                         status=HistoryUnitStatus.DONE,
@@ -154,7 +154,7 @@ class LocalRunner(BaseRunner):
         list_parameters: list[dict],
         history_unit_ids: list[int],
         list_task_files: list[TaskFiles],
-        task_type: Literal["parallel", "compound", "converter_compound"],
+        task_type: MultisubmitTaskType,
         config: LocalBackendConfig,
         user_id: int,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
@@ -197,7 +197,7 @@ class LocalRunner(BaseRunner):
             exceptions = {
                 ind: exception for ind in range(len(list_parameters))
             }
-            if task_type == "parallel":
+            if task_type == TaskType.PARALLEL:
                 with next(get_sync_db()) as db:
                     bulk_update_status_of_history_unit(
                         history_unit_ids=history_unit_ids,
@@ -233,7 +233,7 @@ class LocalRunner(BaseRunner):
                         positional_index
                     ]
                     exceptions[positional_index] = TaskExecutionError(str(e))
-                    if task_type == "parallel":
+                    if task_type == TaskType.PARALLEL:
                         with next(get_sync_db()) as db:
                             update_status_of_history_unit(
                                 history_unit_id=current_history_unit_id,
@@ -252,14 +252,14 @@ class LocalRunner(BaseRunner):
                 with next(get_sync_db()) as db:
                     for positional_index, fut in finished_futures:
                         active_futures.pop(positional_index)
-                        if task_type == "parallel":
+                        if task_type == TaskType.PARALLEL:
                             current_history_unit_id = history_unit_ids[
                                 positional_index
                             ]
 
                         try:
                             results[positional_index] = fut.result()
-                            if task_type == "parallel":
+                            if task_type == TaskType.PARALLEL:
                                 update_status_of_history_unit(
                                     history_unit_id=current_history_unit_id,
                                     status=HistoryUnitStatus.DONE,
@@ -275,7 +275,7 @@ class LocalRunner(BaseRunner):
                             exceptions[positional_index] = TaskExecutionError(
                                 str(e)
                             )
-                            if task_type == "parallel":
+                            if task_type == TaskType.PARALLEL:
                                 update_status_of_history_unit(
                                     history_unit_id=current_history_unit_id,
                                     status=HistoryUnitStatus.FAILED,

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -19,6 +19,8 @@ from fractal_server.app.models.v2 import AccountingRecordSlurm
 from fractal_server.app.runner.exceptions import JobExecutionError
 from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.runner.executors.base_runner import MultisubmitTaskType
+from fractal_server.app.runner.executors.base_runner import SubmitTaskType
 from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.app.runner.v2.db_tools import (
@@ -26,6 +28,7 @@ from fractal_server.app.runner.v2.db_tools import (
 )
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import TaskType
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
@@ -501,12 +504,7 @@ class BaseSlurmRunner(BaseRunner):
         history_unit_id: int,
         task_files: TaskFiles,
         config: SlurmConfig,
-        task_type: Literal[
-            "non_parallel",
-            "converter_non_parallel",
-            "compound",
-            "converter_compound",
-        ],
+        task_type: SubmitTaskType,
         user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
@@ -604,8 +602,8 @@ class BaseSlurmRunner(BaseRunner):
                             )
                         else:
                             if task_type not in [
-                                "compound",
-                                "converter_compound",
+                                TaskType.COMPOUND,
+                                TaskType.CONVERTER_COMPOUND,
                             ]:
                                 update_status_of_history_unit(
                                     history_unit_id=history_unit_id,
@@ -641,7 +639,7 @@ class BaseSlurmRunner(BaseRunner):
         list_parameters: list[dict],
         history_unit_ids: list[int],
         list_task_files: list[TaskFiles],
-        task_type: Literal["parallel", "compound", "converter_compound"],
+        task_type: MultisubmitTaskType,
         config: SlurmConfig,
         user_id: int,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
@@ -654,7 +652,7 @@ class BaseSlurmRunner(BaseRunner):
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
         try:
             if self.is_shutdown():
-                if task_type == "parallel":
+                if task_type == TaskType.PARALLEL:
                     with next(get_sync_db()) as db:
                         bulk_update_status_of_history_unit(
                             history_unit_ids=history_unit_ids,
@@ -680,7 +678,7 @@ class BaseSlurmRunner(BaseRunner):
             workdir_remote = list_task_files[0].wftask_subfolder_remote
 
             # Create local&remote task subfolders
-            if task_type == "parallel":
+            if task_type == TaskType.PARALLEL:
                 self._mkdir_local_folder(workdir_local.as_posix())
                 self._mkdir_remote_folder(folder=workdir_remote.as_posix())
 
@@ -758,7 +756,7 @@ class BaseSlurmRunner(BaseRunner):
                 f" Original error {str(e)}"
             )
             self.scancel_jobs()
-            if task_type == "parallel":
+            if task_type == TaskType.PARALLEL:
                 with next(get_sync_db()) as db:
                     bulk_update_status_of_history_unit(
                         history_unit_ids=history_unit_ids,
@@ -824,7 +822,7 @@ class BaseSlurmRunner(BaseRunner):
                         # `result is None` is not relevant for this purpose.
                         if exception is not None:
                             exceptions[task.index] = exception
-                            if task_type == "parallel":
+                            if task_type == TaskType.PARALLEL:
                                 update_status_of_history_unit(
                                     history_unit_id=history_unit_ids[
                                         task.index
@@ -834,7 +832,7 @@ class BaseSlurmRunner(BaseRunner):
                                 )
                         else:
                             results[task.index] = result
-                            if task_type == "parallel":
+                            if task_type == TaskType.PARALLEL:
                                 update_status_of_history_unit(
                                     history_unit_id=history_unit_ids[
                                         task.index

fractal_server/app/runner/v2/runner.py

@@ -10,10 +10,6 @@ from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import delete
 from sqlmodel import update
 
-from ....images import SingleImage
-from ....images.tools import filter_image_list
-from ....images.tools import find_image_by_zarr_url
-from ..exceptions import JobExecutionError
 from .merge_outputs import merge_outputs
 from .runner_functions import run_v2_task_compound
 from .runner_functions import run_v2_task_non_parallel
@@ -28,13 +24,18 @@ from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.app.runner.exceptions import JobExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_run
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import TaskDumpV2
 from fractal_server.app.schemas.v2 import TaskGroupDumpV2
+from fractal_server.app.schemas.v2 import TaskType
+from fractal_server.images import SingleImage
 from fractal_server.images.status_tools import enrich_images_sync
 from fractal_server.images.status_tools import IMAGE_STATUS_KEY
+from fractal_server.images.tools import filter_image_list
+from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import merge_type_filters
 from fractal_server.types import AttributeFilters
 
@@ -131,7 +132,11 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Filter images by types and attributes (in two steps)
-        if wftask.task_type in ["compound", "parallel", "non_parallel"]:
+        if wftask.task_type in [
+            TaskType.COMPOUND,
+            TaskType.PARALLEL,
+            TaskType.NON_PARALLEL,
+        ]:
             # Non-converter task
             type_filters = copy(current_dataset_type_filters)
             type_filters_patch = merge_type_filters(
@@ -190,7 +195,10 @@ def execute_tasks_v2(
 
         # TASK EXECUTION (V2)
         try:
-            if task.type in ["non_parallel", "converter_non_parallel"]:
+            if task.type in [
+                TaskType.NON_PARALLEL,
+                TaskType.CONVERTER_NON_PARALLEL,
+            ]:
                 outcomes_dict, num_tasks = run_v2_task_non_parallel(
                     images=filtered_images,
                     zarr_dir=zarr_dir,
@@ -205,7 +213,7 @@ def execute_tasks_v2(
                     user_id=user_id,
                     task_type=task.type,
                 )
-            elif task.type == "parallel":
+            elif task.type == TaskType.PARALLEL:
                 outcomes_dict, num_tasks = run_v2_task_parallel(
                     images=filtered_images,
                     wftask=wftask,
@@ -218,7 +226,10 @@ def execute_tasks_v2(
                     dataset_id=dataset.id,
                     user_id=user_id,
                 )
-            elif task.type in ["compound", "converter_compound"]:
+            elif task.type in [
+                TaskType.COMPOUND,
+                TaskType.CONVERTER_COMPOUND,
+            ]:
                 outcomes_dict, num_tasks = run_v2_task_compound(
                     images=filtered_images,
                     zarr_dir=zarr_dir,

fractal_server/app/runner/v2/runner_functions.py

@@ -31,6 +31,7 @@ from fractal_server.app.runner.v2.task_interface import (
     _cast_and_validate_TaskOutput,
 )
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.app.schemas.v2 import TaskType
 from fractal_server.logger import set_logger
 
 __all__ = [
@@ -135,14 +136,17 @@ def run_v2_task_non_parallel(
     ],
     dataset_id: int,
     history_run_id: int,
-    task_type: Literal["non_parallel", "converter_non_parallel"],
+    task_type: Literal[TaskType.NON_PARALLEL, TaskType.CONVERTER_NON_PARALLEL],
     user_id: int,
 ) -> tuple[dict[int, SubmissionOutcome], int]:
     """
     This runs server-side (see `executor` argument)
     """
 
-    if task_type not in ["non_parallel", "converter_non_parallel"]:
+    if task_type not in [
+        TaskType.NON_PARALLEL,
+        TaskType.CONVERTER_NON_PARALLEL,
+    ]:
         raise ValueError(
             f"Invalid {task_type=} for `run_v2_task_non_parallel`."
         )
@@ -166,14 +170,14 @@ def run_v2_task_non_parallel(
         "zarr_dir": zarr_dir,
         **(wftask.args_non_parallel or {}),
     }
-    if task_type == "non_parallel":
+    if task_type == TaskType.NON_PARALLEL:
         function_kwargs["zarr_urls"] = [img["zarr_url"] for img in images]
 
     # Database History operations
     with next(get_sync_db()) as db:
-        if task_type == "non_parallel":
+        if task_type == TaskType.NON_PARALLEL:
             zarr_urls = function_kwargs["zarr_urls"]
-        elif task_type == "converter_non_parallel":
+        elif task_type == TaskType.CONVERTER_NON_PARALLEL:
             zarr_urls = []
 
         history_unit = HistoryUnit(
@@ -388,7 +392,7 @@ def run_v2_task_compound(
     ],
     dataset_id: int,
     history_run_id: int,
-    task_type: Literal["compound", "converter_compound"],
+    task_type: Literal[TaskType.COMPOUND, TaskType.CONVERTER_COMPOUND],
     user_id: int,
 ) -> tuple[dict[int, SubmissionOutcome], int]:
     # Get TaskFiles object
@@ -410,10 +414,10 @@ def run_v2_task_compound(
         "zarr_dir": zarr_dir,
         **(wftask.args_non_parallel or {}),
     }
-    if task_type == "compound":
+    if task_type == TaskType.COMPOUND:
         function_kwargs["zarr_urls"] = [img["zarr_url"] for img in images]
         input_image_zarr_urls = function_kwargs["zarr_urls"]
-    elif task_type == "converter_compound":
+    elif task_type == TaskType.CONVERTER_COMPOUND:
         input_image_zarr_urls = []
 
     # Create database History entries

fractal_server/app/schemas/v2/__init__.py

@@ -31,6 +31,7 @@ from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
 from .task import TaskImportV2Legacy  # noqa F401
 from .task import TaskReadV2  # noqa F401
+from .task import TaskType  # noqa F401
 from .task import TaskUpdateV2  # noqa F401
 from .task_collection import TaskCollectCustomV2  # noqa F401
 from .task_collection import TaskCollectPipV2  # noqa F401

fractal_server/app/schemas/v2/dumps.py

@@ -11,7 +11,7 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
 
-from .task import TaskTypeType
+from .task import TaskType
 from .task_group import TaskGroupV2OriginEnum
 
 
@@ -25,7 +25,7 @@ class ProjectDumpV2(BaseModel):
 class TaskDumpV2(BaseModel):
     id: int
     name: str
-    type: TaskTypeType
+    type: TaskType
 
     command_non_parallel: str | None = None
     command_parallel: str | None = None

fractal_server/app/schemas/v2/manifest.py

@@ -4,6 +4,7 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import model_validator
 
+from .task import TaskType
 from fractal_server.types import DictStrAny
 from fractal_server.types import HttpUrlStr
 from fractal_server.types import NonEmptyStr
@@ -55,15 +56,7 @@ class TaskManifestV2(BaseModel):
     modality: str | None = None
     tags: list[str] = Field(default_factory=list)
 
-    type: None | (
-        Literal[
-            "compound",
-            "converter_compound",
-            "non_parallel",
-            "converter_non_parallel",
-            "parallel",
-        ]
-    ) = None
+    type: None | TaskType = None
 
     @model_validator(mode="after")
     def validate_executable_args_meta(self):

fractal_server/app/schemas/v2/task.py

@@ -1,5 +1,5 @@
+from enum import StrEnum
 from typing import Any
-from typing import Literal
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
@@ -14,18 +14,22 @@ from fractal_server.types import ListUniqueNonEmptyString
 from fractal_server.types import NonEmptyStr
 from fractal_server.types import TypeFilters
 
-TaskTypeType = Literal[
-    "compound",
-    "converter_compound",
-    "non_parallel",
-    "converter_non_parallel",
-    "parallel",
-]
-
 
 logger = set_logger(__name__)
 
 
+class TaskType(StrEnum):
+    """
+    Define the available task types.
+    """
+
+    COMPOUND = "compound"
+    CONVERTER_COMPOUND = "converter_compound"
+    NON_PARALLEL = "non_parallel"
+    CONVERTER_NON_PARALLEL = "converter_non_parallel"
+    PARALLEL = "parallel"
+
+
 class TaskCreateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -51,7 +55,7 @@ class TaskCreateV2(BaseModel):
     tags: ListUniqueNonEmptyString = Field(default_factory=list)
     authors: NonEmptyStr | None = None
 
-    type: TaskTypeType | None = None
+    type: TaskType | None = None
 
     @model_validator(mode="after")
     def validate_commands(self):
@@ -78,11 +82,11 @@ class TaskCreateV2(BaseModel):
                 "Please move to `fractal-task-tools`."
             )
             if self.command_non_parallel is None:
-                self.type = "parallel"
+                self.type = TaskType.PARALLEL
             elif self.command_parallel is None:
-                self.type = "non_parallel"
+                self.type = TaskType.NON_PARALLEL
             else:
-                self.type = "compound"
+                self.type = TaskType.COMPOUND
 
         return self
 
@@ -90,7 +94,7 @@ class TaskCreateV2(BaseModel):
 class TaskReadV2(BaseModel):
     id: int
     name: str
-    type: TaskTypeType
+    type: TaskType
     source: str | None = None
     version: str | None = None
 

fractal_server/app/schemas/v2/workflowtask.py

@@ -9,7 +9,7 @@ from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
-from .task import TaskTypeType
+from .task import TaskType
 from fractal_server.types import DictStrAny
 from fractal_server.types import TypeFilters
 from fractal_server.types import WorkflowTaskArgument
@@ -45,7 +45,7 @@ class WorkflowTaskReadV2(BaseModel):
 
     type_filters: dict[str, bool]
 
-    task_type: TaskTypeType
+    task_type: TaskType
     task_id: int
     task: TaskReadV2
 

fractal_server/exceptions.py

@@ -0,0 +1,2 @@
+class UnreachableBranchError(RuntimeError):
+    pass

fractal_server/utils.py

@@ -13,12 +13,10 @@
 This module provides general purpose utilities that are not specific to any
 subsystem.
 """
-import asyncio
 import shlex
 import subprocess  # nosec
 from datetime import datetime
 from datetime import timezone
-from pathlib import Path
 
 from .logger import get_logger
 from .string_tools import validate_cmd
@@ -31,53 +29,6 @@ def get_timestamp() -> datetime:
     return datetime.now(tz=timezone.utc)
 
 
-async def execute_command_async(
-    *,
-    command: str,
-    cwd: Path | None = None,
-    logger_name: str | None = None,
-) -> str:
-    """
-    Execute arbitrary command
-
-    If the command returns a return code different from zero, a RuntimeError
-    containing the stderr is raised.
-
-    Args:
-        cwd:
-            The working directory for the command execution.
-        command:
-            The command to execute.
-
-    Returns:
-        stdout:
-            The stdout from the command execution.
-
-    Raises:
-        RuntimeError: if the process exited with non-zero status. The error
-            string is set to the `stderr` of the process.
-    """
-    command_split = shlex.split(command)
-    cmd, *args = command_split
-
-    logger = get_logger(logger_name)
-    cwd_kwarg = dict() if cwd is None else dict(cwd=cwd)
-    proc = await asyncio.create_subprocess_exec(
-        cmd,
-        *args,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-        **cwd_kwarg,
-    )
-    stdout, stderr = await proc.communicate()
-    logger.debug(f"Subprocess call to: {command}")
-    logger.debug(stdout.decode("utf-8"))
-    logger.debug(stderr.decode("utf-8"))
-    if proc.returncode != 0:
-        raise RuntimeError(stderr.decode("utf-8"))
-    return stdout.decode("utf-8")
-
-
 def execute_command_sync(
     *,
     command: str,

{fractal_server-2.14.12.dist-info → fractal_server-2.14.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.12
+Version: 2.14.14
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin