fractal-server 2.0.0a10__py3-none-any.whl → 2.0.0a12__py3-none-any.whl

fractal_server/app/runner/v2/runner.py

@@ -12,6 +12,7 @@ from ....images import SingleImage
 from ....images.tools import filter_image_list
 from ....images.tools import find_image_by_zarr_url
 from ....images.tools import match_filter
+from ..exceptions import JobExecutionError
 from ..filenames import FILTERS_FILENAME
 from ..filenames import HISTORY_FILENAME
 from ..filenames import IMAGES_FILENAME
@@ -20,13 +21,12 @@ from .runner_functions import run_v1_task_parallel
 from .runner_functions import run_v2_task_compound
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
+from .task_interface import TaskOutput
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.schemas.v2.dataset import _DatasetHistoryItemV2
 from fractal_server.app.schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
 
-# FIXME: define RESERVED_ARGUMENTS = [", ...]
-
 
 def execute_tasks_v2(
     wf_task_list: list[WorkflowTaskV2],
@@ -52,8 +52,15 @@ def execute_tasks_v2(
     for wftask in wf_task_list:
         task = wftask.task
         task_legacy = wftask.task_legacy
-        task_name = task_legacy.name if wftask.is_legacy_task else task.name
-        logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
+        if wftask.is_legacy_task:
+            task_name = task_legacy.name
+            logger.debug(
+                f"SUBMIT {wftask.order}-th task "
+                f'(legacy, name="{task_name}")'
+            )
+        else:
+            task_name = task.name
+            logger.debug(f'SUBMIT {wftask.order}-th task (name="{task_name}")')
 
         # PRE TASK EXECUTION
 
@@ -72,9 +79,11 @@ def execute_tasks_v2(
         if not wftask.is_legacy_task:
             for image in filtered_images:
                 if not match_filter(image, Filters(types=task.input_types)):
-                    raise ValueError(
-                        f"Filtered images include {image}, which does "
-                        f"not comply with {task.input_types=}."
+                    raise JobExecutionError(
+                        "Invalid filtered image list\n"
+                        f"Task input types: {task.input_types=}\n"
+                        f'Image zarr_url: {image["zarr_url"]}\n'
+                        f'Image types: {image["types"]}\n'
                     )
 
         # TASK EXECUTION (V2)
@@ -115,7 +124,7 @@ def execute_tasks_v2(
                     submit_setup_call=submit_setup_call,
                 )
             else:
-                raise ValueError(f"Invalid {task.type=}.")
+                raise ValueError(f"Unexpected error: Invalid {task.type=}.")
         # TASK EXECUTION (V1)
         else:
             current_task_output = run_v1_task_parallel(
@@ -131,21 +140,36 @@ def execute_tasks_v2(
 
         # POST TASK EXECUTION
 
+        # If `current_task_output` includes no images (to be created, edited or
+        # removed), then flag all the input images as modified. See
+        # fractal-server issue #1374.
+        if (
+            current_task_output.image_list_updates == []
+            and current_task_output.image_list_removals == []
+        ):
+            current_task_output = TaskOutput(
+                **current_task_output.dict(exclude={"image_list_updates"}),
+                image_list_updates=[
+                    dict(zarr_url=img["zarr_url"]) for img in filtered_images
+                ],
+            )
+
         # Update image list
         current_task_output.check_zarr_urls_are_unique()
         for image_obj in current_task_output.image_list_updates:
             image = image_obj.dict()
             # Edit existing image
-            if image["zarr_url"] in [
-                _image["zarr_url"] for _image in tmp_images
-            ]:
+            tmp_image_paths = [img["zarr_url"] for img in tmp_images]
+            if image["zarr_url"] in tmp_image_paths:
                 if (
                     image["origin"] is not None
                     and image["origin"] != image["zarr_url"]
                 ):
-                    raise ValueError(
-                        f"Trying to edit an image with {image['zarr_url']=} "
-                        f"and {image['origin']=}."
+                    raise JobExecutionError(
+                        "Cannot edit an image with zarr_url different from "
+                        "origin.\n"
+                        f"zarr_url={image['zarr_url']}\n"
+                        f"origin={image['origin']}"
                     )
                 img_search = find_image_by_zarr_url(
                     images=tmp_images,
@@ -153,6 +177,7 @@ def execute_tasks_v2(
                 )
                 if img_search is None:
                     raise ValueError(
+                        "Unexpected error: "
                         f"Image with zarr_url {image['zarr_url']} not found, "
                         "while updating image list."
                     )
@@ -188,14 +213,19 @@ def execute_tasks_v2(
             else:
                 # Check that image['zarr_url'] is relative to zarr_dir
                 if not image["zarr_url"].startswith(zarr_dir):
-                    raise ValueError(
-                        f"{zarr_dir} is not a parent directory of "
-                        f"{image['zarr_url']}"
+                    raise JobExecutionError(
+                        "Cannot create image if zarr_dir is not a parent "
+                        "directory of zarr_url.\n"
+                        f"zarr_dir: {zarr_dir}\n"
+                        f"zarr_url: {image['zarr_url']}"
                     )
                 # Check that image['zarr_url'] is not equal to zarr_dir
                 if image["zarr_url"] == zarr_dir:
-                    raise ValueError(
-                        "image['zarr_url'] cannot be equal to zarr_dir"
+                    raise JobExecutionError(
+                        "Cannot create image if zarr_url is equal to "
+                        "zarr_dir.\n"
+                        f"zarr_dir: {zarr_dir}\n"
+                        f"zarr_url: {image['zarr_url']}"
                     )
                 # Propagate attributes and types from `origin` (if any)
                 updated_attributes = {}
@@ -236,8 +266,8 @@ def execute_tasks_v2(
                 images=tmp_images, zarr_url=img_zarr_url
             )
             if img_search is None:
-                raise ValueError(
-                    f"Cannot remove missing image with zarr_url {img_zarr_url}"
+                raise JobExecutionError(
+                    f"Cannot remove missing image (zarr_url={img_zarr_url})."
                 )
             else:
                 tmp_images.pop(img_search["index"])
@@ -249,24 +279,31 @@ def execute_tasks_v2(
                 current_task_output.filters.attributes
             )
 
-        # Update filters.types: current + (task_output + task_manifest)
+        # Find manifest ouptut types
         if wftask.is_legacy_task:
             types_from_manifest = {}
         else:
             types_from_manifest = task.output_types
+
+        # Find task-output types
         if current_task_output.filters is not None:
             types_from_task = current_task_output.filters.types
         else:
             types_from_task = {}
+
         # Check that key sets are disjoint
         set_types_from_manifest = set(types_from_manifest.keys())
         set_types_from_task = set(types_from_task.keys())
         if not set_types_from_manifest.isdisjoint(set_types_from_task):
             overlap = set_types_from_manifest.intersection(set_types_from_task)
-            raise ValueError(
-                "Both task and task manifest did set the same"
-                f"output type. Overlapping keys: {overlap}."
+            raise JobExecutionError(
+                "Some type filters are being set twice, "
+                f"for task '{task_name}'.\n"
+                f"Types from task output: {types_from_task}\n"
+                f"Types from task maniest: {types_from_manifest}\n"
+                f"Overlapping keys: {overlap}"
             )
+
         # Update filters.types
         tmp_filters["types"].update(types_from_manifest)
         tmp_filters["types"].update(types_from_task)

fractal_server/app/runner/v2/runner_functions.py

@@ -8,6 +8,9 @@ from typing import Callable
 from typing import Literal
 from typing import Optional
 
+from pydantic import ValidationError
+
+from ..exceptions import JobExecutionError
 from .deduplicate_list import deduplicate_list
 from .merge_outputs import merge_outputs
 from .runner_functions_low_level import run_single_task
@@ -31,6 +34,34 @@ __all__ = [
 MAX_PARALLELIZATION_LIST_SIZE = 20_000
 
 
+def _cast_and_validate_TaskOutput(
+    task_output: dict[str, Any]
+) -> Optional[TaskOutput]:
+    try:
+        validated_task_output = TaskOutput(**task_output)
+        return validated_task_output
+    except ValidationError as e:
+        raise JobExecutionError(
+            "Validation of task output failed.\n"
+            f"Original error: {str(e)}\n"
+            f"Original data: {task_output}."
+        )
+
+
+def _cast_and_validate_InitTaskOutput(
+    init_task_output: dict[str, Any],
+) -> Optional[InitTaskOutput]:
+    try:
+        validated_init_task_output = InitTaskOutput(**init_task_output)
+        return validated_init_task_output
+    except ValidationError as e:
+        raise JobExecutionError(
+            "Validation of init-task output failed.\n"
+            f"Original error: {str(e)}\n"
+            f"Original data: {init_task_output}."
+        )
+
+
 def no_op_submit_setup_call(
     *,
     wftask: WorkflowTaskV2,
@@ -71,7 +102,7 @@ def _get_executor_options(
 
 def _check_parallelization_list_size(my_list):
     if len(my_list) > MAX_PARALLELIZATION_LIST_SIZE:
-        raise ValueError(
+        raise JobExecutionError(
             "Too many parallelization items.\n"
             f"   {len(my_list)}\n"
             f"   {MAX_PARALLELIZATION_LIST_SIZE=}\n"
@@ -126,12 +157,10 @@ def run_v2_task_non_parallel(
         **executor_options,
     )
     output = future.result()
-    # FIXME V2: handle validation errors
     if output is None:
         return TaskOutput()
     else:
-        validated_output = TaskOutput(**output)
-        return validated_output
+        return _cast_and_validate_TaskOutput(output)
 
 
 def run_v2_task_parallel(
@@ -188,9 +217,7 @@ def run_v2_task_parallel(
         if output is None:
             outputs[ind] = TaskOutput()
         else:
-            # FIXME: improve handling of validation errors
-            validated_output = TaskOutput(**output)
-            outputs[ind] = validated_output
+            outputs[ind] = _cast_and_validate_TaskOutput(output)
 
     merged_output = merge_outputs(outputs)
     return merged_output
@@ -245,7 +272,7 @@ def run_v2_task_compound(
     if output is None:
         init_task_output = InitTaskOutput()
     else:
-        init_task_output = InitTaskOutput(**output)
+        init_task_output = _cast_and_validate_InitTaskOutput(output)
     parallelization_list = init_task_output.parallelization_list
     parallelization_list = deduplicate_list(parallelization_list)
 
@@ -285,8 +312,7 @@ def run_v2_task_compound(
         if output is None:
             outputs[ind] = TaskOutput()
         else:
-            # FIXME: improve handling of validation errors
-            validated_output = TaskOutput(**output)
+            validated_output = _cast_and_validate_TaskOutput(output)
             outputs[ind] = validated_output
 
     merged_output = merge_outputs(outputs)

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -76,19 +76,12 @@ def run_single_task(
         workflow_dir_user = workflow_dir
 
     component = args.pop(_COMPONENT_KEY_, None)
-    if component is None:
-        task_files = get_task_file_paths(
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
-            task_order=wftask.order,
-        )
-    else:
-        task_files = get_task_file_paths(
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
-            task_order=wftask.order,
-            component=component,
-        )
+    task_files = get_task_file_paths(
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        task_order=wftask.order,
+        component=component,
+    )
 
     # Write arguments to args.json file
     with task_files.args.open("w") as f:

fractal_server/app/runner/v2/task_interface.py

@@ -29,8 +29,10 @@ class TaskOutput(BaseModel):
                 if zarr_urls.count(zarr_url) > 1
             ]
             msg = (
-                "TaskOutput image-list updates/removals has "
-                "non-unique zarr_urls:"
+                "TaskOutput "
+                f"({len(self.image_list_updates)} image_list_updates and "
+                f"{len(self.image_list_removals)} image_list_removals) "
+                "has non-unique zarr_urls:"
             )
             for duplicate in duplicates:
                 msg = f"{msg}\n{duplicate}"

fractal_server/app/schemas/__init__.py

@@ -1,4 +1 @@
-from .state import *  # noqa: F401, F403
 from .user import *  # noqa: F401, F403
-from .v1 import *  # noqa: F401, F403
-from .v2 import *  # noqa: F401, F403

fractal_server/app/schemas/v1/__init__.py

@@ -1,12 +1,6 @@
 """
 Schemas for API request/response bodies
 """
-from ..state import _StateBase  # noqa: F401
-from ..state import StateRead  # noqa: F401
-from ..user import UserCreate  # noqa: F401
-from ..user import UserRead  # noqa: F401
-from ..user import UserUpdate  # noqa: F401
-from ..user import UserUpdateStrict  # noqa: F401
 from .applyworkflow import ApplyWorkflowCreateV1  # noqa: F401
 from .applyworkflow import ApplyWorkflowReadV1  # noqa: F401
 from .applyworkflow import ApplyWorkflowUpdateV1  # noqa: F401

fractal_server/app/schemas/v2/__init__.py

@@ -1,4 +1,6 @@
 from .dataset import DatasetCreateV2  # noqa F401
+from .dataset import DatasetExportV2  # noqa F401
+from .dataset import DatasetImportV2  # noqa F401
 from .dataset import DatasetReadV2  # noqa F401
 from .dataset import DatasetUpdateV2  # noqa F401
 from .dumps import DatasetDumpV2  # noqa F401

fractal_server/app/schemas/v2/dataset.py

@@ -103,6 +103,11 @@ class DatasetImportV2(BaseModel):
     images: list[SingleImage] = Field(default_factory=[])
     filters: Filters = Field(default_factory=Filters)
 
+    # Validators
+    @validator("zarr_dir")
+    def normalize_zarr_dir(cls, v: str) -> str:
+        return normalize_url(v)
+
 
 class DatasetExportV2(BaseModel):
     """

fractal_server/app/schemas/v2/workflowtask.py

@@ -12,12 +12,14 @@ from .._validators import valdictkeys
 from .._validators import valint
 from ..v1.task import TaskExportV1
 from ..v1.task import TaskImportV1
-from ..v1.task import TaskReadV1
 from .task import TaskExportV2
 from .task import TaskImportV2
+from .task import TaskLegacyReadV2
 from .task import TaskReadV2
 from fractal_server.images import Filters
 
+RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
+
 
 class WorkflowTaskStatusTypeV2(str, Enum):
     """
@@ -56,14 +58,36 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
         valdictkeys("meta_parallel")
     )
-    _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
-        valdictkeys("args_non_parallel")
-    )
-    _args_parallel = validator("args_parallel", allow_reuse=True)(
-        valdictkeys("args_parallel")
-    )
     _order = validator("order", allow_reuse=True)(valint("order", min_val=0))
 
+    @validator("args_non_parallel")
+    def validate_args_non_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_non_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @validator("args_parallel")
+    def validate_args_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
     @root_validator
     def validate_legacy_task(cls, values):
         if values["is_legacy_task"] and (
@@ -96,7 +120,7 @@ class WorkflowTaskReadV2(BaseModel):
     task_id: Optional[int]
     task: Optional[TaskReadV2]
     task_legacy_id: Optional[int]
-    task_legacy: Optional[TaskReadV1]
+    task_legacy: Optional[TaskLegacyReadV2]
 
 
 class WorkflowTaskUpdateV2(BaseModel):
@@ -114,12 +138,34 @@ class WorkflowTaskUpdateV2(BaseModel):
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
         valdictkeys("meta_parallel")
     )
-    _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
-        valdictkeys("args_non_parallel")
-    )
-    _args_parallel = validator("args_parallel", allow_reuse=True)(
-        valdictkeys("args_parallel")
-    )
+
+    @validator("args_non_parallel")
+    def validate_args_non_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_non_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @validator("args_parallel")
+    def validate_args_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
 
 
 class WorkflowTaskImportV2(BaseModel):

fractal_server/images/tools.py

@@ -33,7 +33,6 @@ def find_image_by_zarr_url(
     return dict(image=copy(images[ind]), index=ind)
 
 
-# FIXME: what is filters
 def match_filter(image: dict[str, Any], filters: Filters) -> bool:
     """
     Find whether an image matches a filter set.

fractal_server/migrations/versions/{80e12e1bc4fd_v2.py → 5bf02391cfef_v2.py}

@@ -1,8 +1,8 @@
-"""V2
+"""v2
 
-Revision ID: 80e12e1bc4fd
+Revision ID: 5bf02391cfef
 Revises: 9fd26a2b0de4
-Create Date: 2024-04-12 10:13:58.085788
+Create Date: 2024-04-18 10:35:19.067833
 
 """
 import sqlalchemy as sa
@@ -11,7 +11,7 @@ from alembic import op
 
 
 # revision identifiers, used by Alembic.
-revision = "80e12e1bc4fd"
+revision = "5bf02391cfef"
 down_revision = "9fd26a2b0de4"
 branch_labels = None
 depends_on = None
@@ -19,6 +19,13 @@ depends_on = None
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "collectionstatev2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("data", sa.JSON(), nullable=True),
+        sa.Column("timestamp", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
     op.create_table(
         "projectv2",
         sa.Column("id", sa.Integer(), nullable=False),
@@ -234,4 +241,5 @@ def downgrade() -> None:
     op.drop_table("datasetv2")
     op.drop_table("taskv2")
     op.drop_table("projectv2")
+    op.drop_table("collectionstatev2")
     # ### end Alembic commands ###

fractal_server/tasks/v1/background_operations.py

@@ -15,8 +15,8 @@ from ..utils import slugify_task_name
 from ._TaskCollectPip import _TaskCollectPip
 from fractal_server.app.db import DBSyncSession
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.models import State
-from fractal_server.app.models import Task
+from fractal_server.app.models.v1 import State
+from fractal_server.app.models.v1 import Task
 from fractal_server.app.schemas.v1 import TaskCollectStatusV1
 from fractal_server.app.schemas.v1 import TaskCreateV1
 from fractal_server.app.schemas.v1 import TaskReadV1

fractal_server/tasks/v2/background_operations.py

@@ -15,7 +15,7 @@ from ..utils import slugify_task_name
 from ._TaskCollectPip import _TaskCollectPip
 from fractal_server.app.db import DBSyncSession
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.models import State
+from fractal_server.app.models.v2 import CollectionStateV2
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.schemas.v2 import TaskCollectStatusV2
 from fractal_server.app.schemas.v2 import TaskCreateV2
@@ -313,7 +313,7 @@ async def background_collect_pip(
         logger.debug(f"{key}: {value}")
 
     with next(get_sync_db()) as db:
-        state: State = db.get(State, state_id)
+        state: CollectionStateV2 = db.get(CollectionStateV2, state_id)
         data = TaskCollectStatusV2(**state.data)
         data.info = None
 

{fractal_server-2.0.0a10.dist-info → fractal_server-2.0.0a12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.0.0a10
+Version: 2.0.0a12
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause