fractal-server 2.14.0a6__py3-none-any.whl → 2.14.0a8__py3-none-any.whl

fractal_server/app/runner/v2/runner_functions.py

@@ -15,7 +15,6 @@ from .runner_functions_low_level import run_single_task
 from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.history.status_enum import XXXStatus
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
@@ -23,12 +22,15 @@ from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.components import _index_to_component
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.v2._db_tools import bulk_upsert_image_cache_fast
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
 
 
 __all__ = [
-    "run_v2_task_non_parallel",
     "run_v2_task_parallel",
+    "run_v2_task_non_parallel",
     "run_v2_task_compound",
+    "run_v2_task_converter_non_parallel",
+    "run_v2_task_converter_compound",
 ]
 
 MAX_PARALLELIZATION_LIST_SIZE = 20_000
@@ -125,7 +127,7 @@ def run_v2_task_non_parallel(
     with next(get_sync_db()) as db:
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
-            status=XXXStatus.SUBMITTED,
+            status=HistoryUnitStatus.SUBMITTED,
             logfile=None,  # FIXME
             zarr_urls=function_kwargs["zarr_urls"],
         )
@@ -155,6 +157,92 @@ def run_v2_task_non_parallel(
             root_dir_remote=workflow_dir_remote,
         ),
         parameters=function_kwargs,
+        task_type="non_parallel",
+        **executor_options,
+    )
+
+    num_tasks = 1
+    with next(get_sync_db()) as db:
+        if exception is None:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.DONE)
+            )
+            db.commit()
+            if result is None:
+                return (TaskOutput(), num_tasks, {})
+            else:
+                return (_cast_and_validate_TaskOutput(result), num_tasks, {})
+        else:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.FAILED)
+            )
+            db.commit()
+            return (TaskOutput(), num_tasks, {0: exception})
+
+
+def run_v2_task_converter_non_parallel(
+    *,
+    zarr_dir: str,
+    task: TaskV2,
+    wftask: WorkflowTaskV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    executor: BaseRunner,
+    submit_setup_call: callable = no_op_submit_setup_call,
+    dataset_id: int,
+    history_run_id: int,
+) -> tuple[TaskOutput, int, dict[int, BaseException]]:
+    """
+    This runs server-side (see `executor` argument)
+    """
+
+    if workflow_dir_remote is None:
+        workflow_dir_remote = workflow_dir_local
+        logging.warning(
+            "In `run_single_task`, workflow_dir_remote=None. Is this right?"
+        )
+        workflow_dir_remote = workflow_dir_local
+
+    executor_options = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="non_parallel",
+    )
+
+    function_kwargs = {
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: _index_to_component(0),
+        **(wftask.args_non_parallel or {}),
+    }
+
+    # Database History operations
+    with next(get_sync_db()) as db:
+        history_unit = HistoryUnit(
+            history_run_id=history_run_id,
+            status=HistoryUnitStatus.SUBMITTED,
+            logfile=None,  # FIXME
+            zarr_urls=[],
+        )
+        db.add(history_unit)
+        db.commit()
+        db.refresh(history_unit)
+        history_unit_id = history_unit.id
+
+    result, exception = executor.submit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_non_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        task_type="converter_non_parallel",
+        parameters=function_kwargs,
         **executor_options,
     )
 
@@ -164,7 +252,7 @@ def run_v2_task_non_parallel(
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.DONE)
+                .values(status=HistoryUnitStatus.DONE)
             )
             db.commit()
             if result is None:
@@ -175,7 +263,7 @@ def run_v2_task_non_parallel(
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.FAILED)
+                .values(status=HistoryUnitStatus.FAILED)
             )
             db.commit()
             return (TaskOutput(), num_tasks, {0: exception})
@@ -217,7 +305,7 @@ def run_v2_task_parallel(
     history_units = [
         HistoryUnit(
             history_run_id=history_run_id,
-            status=XXXStatus.SUBMITTED,
+            status=HistoryUnitStatus.SUBMITTED,
             logfile=None,  # FIXME
             zarr_urls=[image["zarr_url"]],
         )
@@ -255,6 +343,7 @@ def run_v2_task_parallel(
             root_dir_remote=workflow_dir_remote,
         ),
         list_parameters=list_function_kwargs,
+        task_type="parallel",
         **executor_options,
     )
 
@@ -280,12 +369,12 @@ def run_v2_task_parallel(
         db.execute(
             update(HistoryUnit)
             .where(HistoryUnit.id.in_(history_unit_ids_done))
-            .values(status=XXXStatus.DONE)
+            .values(status=HistoryUnitStatus.DONE)
         )
         db.execute(
             update(HistoryUnit)
             .where(HistoryUnit.id.in_(history_unit_ids_failed))
-            .values(status=XXXStatus.FAILED)
+            .values(status=HistoryUnitStatus.FAILED)
         )
         db.commit()
 
@@ -334,7 +423,7 @@ def run_v2_task_compound(
         # Create a single `HistoryUnit` for the whole compound task
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
-            status=XXXStatus.SUBMITTED,
+            status=HistoryUnitStatus.SUBMITTED,
             logfile=None,  # FIXME
             zarr_urls=input_image_zarr_urls,
         )
@@ -365,6 +454,159 @@ def run_v2_task_compound(
             root_dir_remote=workflow_dir_remote,
         ),
         parameters=function_kwargs,
+        task_type="compound",
+        **executor_options_init,
+    )
+
+    num_tasks = 1
+    if exception is None:
+        if result is None:
+            init_task_output = InitTaskOutput()
+        else:
+            init_task_output = _cast_and_validate_InitTaskOutput(result)
+    else:
+        with next(get_sync_db()) as db:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.FAILED)
+            )
+            db.commit()
+        return (TaskOutput(), num_tasks, {0: exception})
+
+    parallelization_list = init_task_output.parallelization_list
+    parallelization_list = deduplicate_list(parallelization_list)
+
+    num_tasks = 1 + len(parallelization_list)
+
+    # 3/B: parallel part of a compound task
+    _check_parallelization_list_size(parallelization_list)
+
+    if len(parallelization_list) == 0:
+        with next(get_sync_db()) as db:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.DONE)
+            )
+            db.commit()
+        return (TaskOutput(), 0, {})
+
+    list_function_kwargs = [
+        {
+            "zarr_url": parallelization_item.zarr_url,
+            "init_args": parallelization_item.init_args,
+            _COMPONENT_KEY_: f"compute_{_index_to_component(ind)}",
+            **(wftask.args_parallel or {}),
+        }
+        for ind, parallelization_item in enumerate(parallelization_list)
+    ]
+
+    results, exceptions = executor.multisubmit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        list_parameters=list_function_kwargs,
+        task_type="compound",
+        **executor_options_compute,
+    )
+
+    outputs = []
+    failure = False
+    for ind in range(len(list_function_kwargs)):
+        if ind in results.keys():
+            result = results[ind]
+            if result is None:
+                output = TaskOutput()
+            else:
+                output = _cast_and_validate_TaskOutput(result)
+            outputs.append(output)
+
+        elif ind in exceptions.keys():
+            print(f"Bad: {exceptions[ind]}")
+            failure = True
+        else:
+            print("VERY BAD - should have not reached this point")
+
+    with next(get_sync_db()) as db:
+        if failure:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.FAILED)
+            )
+        else:
+            db.execute(
+                update(HistoryUnit)
+                .where(HistoryUnit.id == history_unit_id)
+                .values(status=HistoryUnitStatus.DONE)
+            )
+        db.commit()
+
+    merged_output = merge_outputs(outputs)
+    return (merged_output, num_tasks, exceptions)
+
+
+def run_v2_task_converter_compound(
+    *,
+    zarr_dir: str,
+    task: TaskV2,
+    wftask: WorkflowTaskV2,
+    executor: BaseRunner,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    submit_setup_call: callable = no_op_submit_setup_call,
+    dataset_id: int,
+    history_run_id: int,
+) -> tuple[TaskOutput, int, dict[int, BaseException]]:
+    executor_options_init = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="non_parallel",
+    )
+    executor_options_compute = submit_setup_call(
+        wftask=wftask,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
+        which_type="parallel",
+    )
+
+    # 3/A: non-parallel init task
+    function_kwargs = {
+        "zarr_dir": zarr_dir,
+        _COMPONENT_KEY_: f"init_{_index_to_component(0)}",
+        **(wftask.args_non_parallel or {}),
+    }
+
+    # Create database History entries
+    with next(get_sync_db()) as db:
+        # Create a single `HistoryUnit` for the whole compound task
+        history_unit = HistoryUnit(
+            history_run_id=history_run_id,
+            status=HistoryUnitStatus.SUBMITTED,
+            logfile=None,  # FIXME
+            zarr_urls=[],
+        )
+        db.add(history_unit)
+        db.commit()
+        db.refresh(history_unit)
+        history_unit_id = history_unit.id
+
+    result, exception = executor.submit(
+        functools.partial(
+            run_single_task,
+            wftask=wftask,
+            command=task.command_non_parallel,
+            root_dir_local=workflow_dir_local,
+            root_dir_remote=workflow_dir_remote,
+        ),
+        parameters=function_kwargs,
+        task_type="converter_compound",
         **executor_options_init,
     )
 
@@ -379,7 +621,7 @@ def run_v2_task_compound(
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.FAILED)
+                .values(status=HistoryUnitStatus.FAILED)
             )
             db.commit()
         return (TaskOutput(), num_tasks, {0: exception})
@@ -397,7 +639,7 @@ def run_v2_task_compound(
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.DONE)
+                .values(status=HistoryUnitStatus.DONE)
             )
             db.commit()
         return (TaskOutput(), 0, {})
@@ -421,7 +663,7 @@ def run_v2_task_compound(
             root_dir_remote=workflow_dir_remote,
         ),
         list_parameters=list_function_kwargs,
-        in_compound_task=True,
+        task_type="converter_compound",
         **executor_options_compute,
     )
 
@@ -447,13 +689,13 @@ def run_v2_task_compound(
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.FAILED)
+                .values(status=HistoryUnitStatus.FAILED)
             )
         else:
             db.execute(
                 update(HistoryUnit)
                 .where(HistoryUnit.id == history_unit_id)
-                .values(status=XXXStatus.DONE)
+                .values(status=HistoryUnitStatus.DONE)
             )
         db.commit()
 

fractal_server/app/schemas/v2/__init__.py

@@ -7,8 +7,14 @@ from .dataset import DatasetUpdateV2  # noqa F401
 from .dumps import DatasetDumpV2  # noqa F401
 from .dumps import ProjectDumpV2  # noqa F401
 from .dumps import TaskDumpV2  # noqa F401
+from .dumps import TaskGroupDumpV2  # noqa F401
 from .dumps import WorkflowDumpV2  # noqa F401
 from .dumps import WorkflowTaskDumpV2  # noqa F401
+from .history import HistoryRunReadAggregated  # noqa F401
+from .history import HistoryUnitRead  # noqa F401
+from .history import HistoryUnitStatus  # noqa F401
+from .history import ImageLogsRequest  # noqa F401
+from .history import ZarrUrlAndStatus  # noqa F401
 from .job import JobCreateV2  # noqa F401
 from .job import JobReadV2  # noqa F401
 from .job import JobStatusTypeV2  # noqa F401
@@ -18,6 +24,7 @@ from .manifest import TaskManifestV2  # noqa F401
 from .project import ProjectCreateV2  # noqa F401
 from .project import ProjectReadV2  # noqa F401
 from .project import ProjectUpdateV2  # noqa F401
+from .status_legacy import WorkflowTaskStatusTypeV2  # noqa F401
 from .task import TaskCreateV2  # noqa F401
 from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
@@ -47,5 +54,4 @@ from .workflowtask import WorkflowTaskImportV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2WithWarning  # noqa F401
 from .workflowtask import WorkflowTaskReplaceV2  # noqa F401
-from .workflowtask import WorkflowTaskStatusTypeV2  # noqa F401
 from .workflowtask import WorkflowTaskUpdateV2  # noqa F401

fractal_server/app/schemas/v2/dumps.py

@@ -1,18 +1,20 @@
 """
-
 Dump models differ from their Read counterpart in that:
 * They are directly JSON-able, without any additional encoder.
-* They may only include a subset of the Read attributes.
+* They may include only a subset of the available fields.
 
 These models are used in at least two situations:
 1. In the "*_dump" attributes of Job models;
-2. In the `_DatasetHistoryItem.workflowtask` model, to trim its size.
+2. In the history items, to trim their size.
 """
 from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import Field
 
+from .task import TaskTypeType
+from .task_group import TaskGroupV2OriginEnum
 from fractal_server.images.models import AttributeFiltersType
 
 
@@ -26,7 +28,7 @@ class ProjectDumpV2(BaseModel):
 class TaskDumpV2(BaseModel):
     id: int
     name: str
-    type: str
+    type: TaskTypeType
 
     command_non_parallel: Optional[str] = None
     command_parallel: Optional[str] = None
@@ -72,3 +74,17 @@ class DatasetDumpV2(BaseModel):
     zarr_dir: str
     type_filters: dict[str, bool]
     attribute_filters: AttributeFiltersType
+
+
+class TaskGroupDumpV2(BaseModel):
+    id: int
+    origin: TaskGroupV2OriginEnum
+    pkg_name: str
+    version: Optional[str] = None
+    python_version: Optional[str] = None
+    pip_extras: Optional[str] = None
+    pinned_package_versions: dict[str, str] = Field(default_factory=dict)
+
+    path: Optional[str] = None
+    venv_path: Optional[str] = None
+    wheel_path: Optional[str] = None

fractal_server/app/schemas/v2/history.py

@@ -0,0 +1,54 @@
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from typing import Optional
+
+from pydantic import AwareDatetime
+from pydantic import BaseModel
+from pydantic import field_serializer
+
+
+class HistoryUnitStatus(str, Enum):
+    """
+    Available status for images
+
+    Attributes:
+        SUBMITTED:
+        DONE:
+        FAILED:
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class HistoryUnitRead(BaseModel):
+    id: int
+    logfile: Optional[str] = None
+    status: HistoryUnitStatus
+    zarr_urls: list[str]
+
+
+class HistoryRunReadAggregated(BaseModel):
+    id: int
+    timestamp_started: AwareDatetime
+    workflowtask_dump: dict[str, Any]
+    num_submitted_units: int
+    num_done_units: int
+    num_failed_units: int
+
+    @field_serializer("timestamp_started")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()
+
+
+class ImageLogsRequest(BaseModel):
+    workflowtask_id: int
+    dataset_id: int
+    zarr_url: str
+
+
+class ZarrUrlAndStatus(BaseModel):
+    zarr_url: str
+    status: Optional[HistoryUnitStatus] = None

fractal_server/app/schemas/v2/manifest.py

@@ -1,4 +1,5 @@
 from typing import Any
+from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
@@ -56,6 +57,16 @@ class TaskManifestV2(BaseModel):
     modality: Optional[str] = None
     tags: list[str] = Field(default_factory=list)
 
+    type: Optional[
+        Literal[
+            "compound",
+            "converter_compound",
+            "non_parallel",
+            "converter_non_parallel",
+            "parallel",
+        ]
+    ] = None
+
     @model_validator(mode="after")
     def validate_executable_args_meta(self):
         executable_non_parallel = self.executable_non_parallel

fractal_server/app/schemas/v2/status_legacy.py

@@ -0,0 +1,35 @@
+from enum import Enum
+
+from pydantic import BaseModel
+from pydantic import Field
+
+
+class WorkflowTaskStatusTypeV2(str, Enum):
+    """
+    Define the available values for the status of a `WorkflowTask`.
+
+    This model is used within the `Dataset.history` attribute, which is
+    constructed in the runner and then used in the API (e.g. in the
+    `api/v2/project/{project_id}/dataset/{dataset_id}/status` endpoint).
+
+    Attributes:
+        SUBMITTED: The `WorkflowTask` is part of a running job.
+        DONE: The most-recent execution of this `WorkflowTask` was successful.
+        FAILED: The most-recent execution of this `WorkflowTask` failed.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class LegacyStatusReadV2(BaseModel):
+    """
+    Response type for the
+    `/project/{project_id}/status/` endpoint
+    """
+
+    status: dict[
+        str,
+        WorkflowTaskStatusTypeV2,
+    ] = Field(default_factory=dict)

fractal_server/app/schemas/v2/task.py

@@ -13,8 +13,20 @@ from .._validators import cant_set_none
 from fractal_server.app.schemas._validators import NonEmptyString
 from fractal_server.app.schemas._validators import val_unique_list
 from fractal_server.app.schemas._validators import valdict_keys
+from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
 
+TaskTypeType = Literal[
+    "compound",
+    "converter_compound",
+    "non_parallel",
+    "converter_non_parallel",
+    "parallel",
+]
+
+
+logger = set_logger(__name__)
+
 
 class TaskCreateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
@@ -41,6 +53,8 @@ class TaskCreateV2(BaseModel):
     tags: list[NonEmptyString] = Field(default_factory=list)
     authors: Optional[NonEmptyString] = None
 
+    type: Optional[TaskTypeType] = None
+
     # Validators
 
     @field_validator(
@@ -69,6 +83,23 @@ class TaskCreateV2(BaseModel):
 
         return self
 
+    @model_validator(mode="after")
+    def set_task_type(self):
+        if self.type is None:
+            logger.warning(
+                f"Task type is not set for task '{self.name}', "
+                "which will be deprecated in a future version. "
+                "Please move to `fractal-task-tools`."
+            )
+            if self.command_non_parallel is None:
+                self.type = "parallel"
+            elif self.command_parallel is None:
+                self.type = "non_parallel"
+            else:
+                self.type = "compound"
+
+        return self
+
     _meta_non_parallel = field_validator("meta_non_parallel")(
         classmethod(valdict_keys("meta_non_parallel"))
     )
@@ -104,7 +135,7 @@ class TaskCreateV2(BaseModel):
 class TaskReadV2(BaseModel):
     id: int
     name: str
-    type: Literal["parallel", "non_parallel", "compound"]
+    type: TaskTypeType
     source: Optional[str] = None
     version: Optional[str] = None
 

fractal_server/app/schemas/v2/workflowtask.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -16,29 +15,11 @@ from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
+from .task import TaskTypeType
 
 RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
 
 
-class WorkflowTaskStatusTypeV2(str, Enum):
-    """
-    Define the available values for the status of a `WorkflowTask`.
-
-    This model is used within the `Dataset.history` attribute, which is
-    constructed in the runner and then used in the API (e.g. in the
-    `api/v2/project/{project_id}/dataset/{dataset_id}/status` endpoint).
-
-    Attributes:
-        SUBMITTED: The `WorkflowTask` is part of a running job.
-        DONE: The most-recent execution of this `WorkflowTask` was successful.
-        FAILED: The most-recent execution of this `WorkflowTask` failed.
-    """
-
-    SUBMITTED = "submitted"
-    DONE = "done"
-    FAILED = "failed"
-
-
 class WorkflowTaskCreateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -113,7 +94,7 @@ class WorkflowTaskReadV2(BaseModel):
 
     type_filters: dict[str, bool]
 
-    task_type: str
+    task_type: TaskTypeType
     task_id: int
     task: TaskReadV2