fractal-server 2.0.0a0__tar.gz → 2.0.0a2__tar.gz

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.0.0a0
+Version: 2.0.0a2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.0.0a2/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.0.0a2"

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/models/__init__.py

@@ -1,7 +1,6 @@
 """
 `models` module
 """
-from ..schemas.v1 import *  # noqa F401, F403  # FIXME: remove this
 from .security import *  # noqa: F401, F403
 from .state import State  # noqa: F401
 from .v1 import *  # noqa: F401, F403

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/models/v1/__init__.py

@@ -4,6 +4,7 @@
 from .dataset import Dataset  # noqa: F401
 from .dataset import Resource  # noqa: F401
 from .job import ApplyWorkflow  # noqa: F403, F401
+from .job import JobStatusTypeV1  # noqa: F401, F403
 from .project import Project  # noqa: F403, F401
 from .task import Task  # noqa: F403, F401
 from .workflow import Workflow  # noqa: F401, F403

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/admin/v2.py

@@ -268,7 +268,6 @@ async def flag_task_v1_as_v2_compatible(
         )
 
     task.is_v2_compatible = is_v2_compatible
-    db.add(task)
     await db.commit()
     await db.close()
 

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v1/job.py

@@ -10,9 +10,9 @@ from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import ApplyWorkflow
-from ....models import JobStatusTypeV1
-from ....models import Project
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import JobStatusTypeV1
+from ....models.v1 import Project
 from ....runner.filenames import WORKFLOW_LOG_FILENAME
 from ....schemas.v1 import ApplyWorkflowReadV1
 from ....security import current_active_user

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v2/__init__.py

@@ -3,11 +3,11 @@
 """
 from fastapi import APIRouter
 
-from .apply import router as runner_router_v2
 from .dataset import router as dataset_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
+from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
 from .workflow import router as workflow_router_v2
@@ -19,7 +19,7 @@ router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
-router_api_v2.include_router(runner_router_v2, tags=["V2 Apply-workflow"])
+router_api_v2.include_router(submit_job_router_v2, tags=["V2 Submit Job"])
 router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
 router_api_v2.include_router(
     task_collection_router_v2, prefix="/task", tags=["V2 Task Collection"]

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v2/dataset.py

@@ -186,7 +186,6 @@ async def delete_dataset(
     jobs = res.scalars().all()
     for job in jobs:
         job.dataset_id = None
-        await db.merge(job)
     await db.commit()
 
     # Delete dataset

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v2/images.py

@@ -17,7 +17,7 @@ from fractal_server.app.security import current_active_user
 from fractal_server.app.security import User
 from fractal_server.images import Filters
 from fractal_server.images import SingleImage
-from fractal_server.images.tools import match_filter_SingleImage
+from fractal_server.images.tools import match_filter
 
 router = APIRouter()
 
@@ -68,7 +68,6 @@ async def post_new_image(
     dataset.images.append(new_image.dict())
     flag_modified(dataset, "images")
 
-    await db.merge(dataset)
     await db.commit()
 
     return Response(status_code=status.HTTP_201_CREATED)
@@ -106,9 +105,7 @@ async def query_dataset_images(
         images = [
             image
             for image in images
-            if match_filter_SingleImage(
-                SingleImage(**image), Filters(**dataset.filters)
-            )
+            if match_filter(image, Filters(**dataset.filters))
         ]
 
     attributes = {}
@@ -138,8 +135,8 @@ async def query_dataset_images(
             images = [
                 image
                 for image in images
-                if match_filter_SingleImage(
-                    SingleImage(**image),
+                if match_filter(
+                    image,
                     Filters(**query.filters.dict()),
                 )
             ]
@@ -159,7 +156,6 @@ async def query_dataset_images(
 
     if total_count == 0:
         page = 1
-        page_size = 0
     else:
         last_page = (total_count // page_size) + (total_count % page_size > 0)
         if page > last_page:
@@ -206,7 +202,6 @@ async def delete_dataset_images(
     dataset.images.remove(image_to_remove)
     flag_modified(dataset, "images")
 
-    await db.merge(dataset)
     await db.commit()
 
     return Response(status_code=status.HTTP_204_NO_CONTENT)

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v2/project.py

@@ -167,7 +167,6 @@ async def delete_project(
         jobs = res.scalars().all()
         for job in jobs:
             job.workflow_id = None
-            await db.merge(job)
         # Delete workflow
         await db.delete(wf)
     await db.commit()
@@ -184,7 +183,6 @@ async def delete_project(
         jobs = res.scalars().all()
         for job in jobs:
             job.dataset_id = None
-            await db.merge(job)
         # Delete dataset
         await db.delete(ds)
     await db.commit()
@@ -195,7 +193,6 @@ async def delete_project(
     jobs = res.scalars().all()
     for job in jobs:
         job.project_id = None
-        await db.merge(job)
 
     await db.commit()
 

fractal_server-2.0.0a0/fractal_server/app/routes/api/v2/apply.py → fractal_server-2.0.0a2/fractal_server/app/routes/api/v2/submit.py

@@ -36,7 +36,7 @@ router = APIRouter()
 
 
 @router.post(
-    "/project/{project_id}/workflow/{workflow_id}/apply/",
+    "/project/{project_id}/job/submit/",
     status_code=status.HTTP_202_ACCEPTED,
     response_model=JobReadV2,
 )

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/routes/api/v2/workflow.py

@@ -213,7 +213,6 @@ async def delete_workflow(
     jobs = res.scalars().all()
     for job in jobs:
         job.workflow_id = None
-        await db.merge(job)
     await db.commit()
 
     # Delete workflow

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/executors/slurm/executor.py

@@ -46,6 +46,7 @@ from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
+from fractal_server.app.runner.components import _COMPONENT_KEY_
 
 
 logger = set_logger(__name__)
@@ -544,7 +545,7 @@ class FractalSlurmExecutor(SlurmExecutor):
         single_task_submission: bool = False,
         args: Optional[Sequence[Any]] = None,
         kwargs: Optional[dict] = None,
-        components: list[Any] = None,
+        components: Optional[list[Any]] = None,
     ) -> Future:
         """
         Submit a multi-task job to the pool, where each task is handled via the
@@ -580,6 +581,10 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         # Define slurm-job-related files
         if single_task_submission:
+            if components is not None:
+                raise ValueError(
+                    f"{single_task_submission=} but components is not None"
+                )
             job = SlurmJob(
                 slurm_file_prefix=slurm_file_prefix,
                 num_tasks_tot=1,
@@ -603,15 +608,23 @@ class FractalSlurmExecutor(SlurmExecutor):
                 num_tasks_tot=num_tasks_tot,
                 slurm_config=slurm_config,
             )
-            job.wftask_file_prefixes = tuple(
-                get_task_file_paths(
-                    workflow_dir=task_files.workflow_dir,
-                    workflow_dir_user=task_files.workflow_dir_user,
-                    task_order=task_files.task_order,
-                    component=component,
-                ).file_prefix
-                for component in components
-            )
+
+            _prefixes = []
+            for component in components:
+                if isinstance(component, dict):
+                    # This is needed for V2
+                    actual_component = component.get(_COMPONENT_KEY_, None)
+                else:
+                    actual_component = component
+                _prefixes.append(
+                    get_task_file_paths(
+                        workflow_dir=task_files.workflow_dir,
+                        workflow_dir_user=task_files.workflow_dir_user,
+                        task_order=task_files.task_order,
+                        component=actual_component,
+                    ).file_prefix
+                )
+            job.wftask_file_prefixes = tuple(_prefixes)
 
         # Define I/O pickle file names/paths
         job.input_pickle_files = tuple(

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/task_files.py

@@ -1,4 +1,3 @@
-from functools import lru_cache
 from pathlib import Path
 from typing import Optional
 
@@ -84,7 +83,6 @@ class TaskFiles:
         )
 
 
-@lru_cache()
 def get_task_file_paths(
     workflow_dir: Path,
     workflow_dir_user: Path,

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v1/__init__.py

@@ -34,8 +34,8 @@ from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from ..filenames import WORKFLOW_LOG_FILENAME
-from ..v1._local import process_workflow as local_process_workflow
-from ..v1._slurm import process_workflow as slurm_process_workflow
+from ._local import process_workflow as local_process_workflow
+from ._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
 from .common import validate_workflow_compatibility  # noqa: F401
 from .handle_failed_job import assemble_history_failed_job

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v1/_local/__init__.py

@@ -25,11 +25,11 @@ from typing import Optional
 
 from ....models import Workflow  # FIXME: this is v1 specific
 from ...async_wrap import async_wrap
-from ...executors.local.executor import FractalThreadPoolExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks  # FIXME: this is v1 specific
 from ..common import TaskParameters  # FIXME: this is v1 specific
 from ._submit_setup import _local_submit_setup
+from .executor import FractalThreadPoolExecutor
 
 
 def _process_workflow(

{fractal_server-2.0.0a0/fractal_server/app/runner/executors/local → fractal_server-2.0.0a2/fractal_server/app/runner/v1/_local}/executor.py

@@ -18,8 +18,8 @@ from typing import Iterable
 from typing import Optional
 from typing import Sequence
 
-from ...v1._local._local_config import get_default_local_backend_config
-from ...v1._local._local_config import LocalBackendConfig
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
 
 
 class FractalThreadPoolExecutor(ThreadPoolExecutor):

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v2/__init__.py

@@ -31,7 +31,6 @@ from ._slurm import process_workflow as slurm_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_images_failed_job
-from .runner import execute_tasks_v2  # noqa
 from fractal_server import __VERSION__
 
 _backends = {}

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v2/_local/__init__.py

@@ -25,12 +25,10 @@ from typing import Optional
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
-from ...executors.local.executor import FractalThreadPoolExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
-
-# from typing import Any
+from .executor import FractalThreadPoolExecutor
 
 
 def _process_workflow(

fractal_server-2.0.0a2/fractal_server/app/runner/v2/_local/executor.py

@@ -0,0 +1,100 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Custom version of Python
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor)).
+"""
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+from typing import Iterable
+from typing import Optional
+from typing import Sequence
+
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
+
+
+class FractalThreadPoolExecutor(ThreadPoolExecutor):
+    """
+    Custom version of
+    [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
+    that overrides the `submit` and `map` methods
+    """
+
+    def submit(
+        self,
+        *args,
+        local_backend_config: Optional[LocalBackendConfig] = None,
+        **kwargs,
+    ):
+        """
+        Compared to the `ThreadPoolExecutor` method, here we accept an addition
+        keyword argument (`local_backend_config`), which is then simply
+        ignored.
+        """
+        return super().submit(*args, **kwargs)
+
+    def map(
+        self,
+        fn: Callable,
+        *iterables: Sequence[Iterable],
+        local_backend_config: Optional[LocalBackendConfig] = None,
+    ):
+        """
+        Custom version of the `Executor.map` method
+
+        The main change with the respect to the original `map` method is that
+        the list of tasks to be executed is split into chunks, and then
+        `super().map` is called (sequentially) on each chunk. The goal of this
+        change is to limit parallelism, e.g. due to limited computational
+        resources.
+
+        Other changes from the `concurrent.futures` `map` method:
+
+        1. Removed `timeout` argument;
+        2. Removed `chunksize`;
+        3. All iterators (both inputs and output ones) are transformed into
+           lists.
+
+        Args:
+            fn: A callable function.
+            iterables: The argument iterables (one iterable per argument of
+                       `fn`).
+           local_backend_config: The backend configuration, needed to extract
+                                 `parallel_tasks_per_job`.
+        """
+
+        # Preliminary check
+        iterable_lengths = [len(it) for it in iterables]
+        if not len(set(iterable_lengths)) == 1:
+            raise ValueError("Iterables have different lengths.")
+
+        # Set total number of arguments
+        n_elements = len(iterables[0])
+
+        # Set parallel_tasks_per_job
+        if local_backend_config is None:
+            local_backend_config = get_default_local_backend_config()
+        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
+        if parallel_tasks_per_job is None:
+            parallel_tasks_per_job = n_elements
+
+        # Execute tasks, in chunks of size parallel_tasks_per_job
+        results = []
+        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+            chunk_iterables = [
+                it[ind_chunk : ind_chunk + parallel_tasks_per_job]  # noqa
+                for it in iterables
+            ]
+            map_iter = super().map(fn, *chunk_iterables)
+            results.extend(list(map_iter))
+
+        return iter(results)

fractal_server-2.0.0a2/fractal_server/app/runner/v2/_slurm/__init__.py

@@ -0,0 +1,139 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Slurm Bakend
+
+This backend runs fractal workflows in a SLURM cluster using Clusterfutures
+Executor objects.
+"""
+from pathlib import Path
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.slurm.executor import FractalSlurmExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _slurm_submit_setup
+
+# from .._common import execute_tasks
+# from ..common import async_wrap
+# from ..common import set_start_and_last_task_index
+# from ..common import TaskParameters
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    first_task_index: int,
+    last_task_index: int,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    user_cache_dir: str,
+    worker_init: Optional[Union[str, list[str]]] = None,
+) -> dict[str, Any]:
+    """
+    Internal processing routine for the SLURM backend
+
+    This function initialises the a FractalSlurmExecutor, setting logging,
+    workflow working dir and user to impersonate. It then schedules the
+    workflow tasks and returns the new dataset attributes
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+
+    Returns:
+        new_dataset_attributes:
+    """
+
+    if not slurm_user:
+        raise RuntimeError(
+            "slurm_user argument is required, for slurm backend"
+        )
+
+    if isinstance(worker_init, str):
+        worker_init = worker_init.split("\n")
+
+    with FractalSlurmExecutor(
+        debug=True,
+        keep_logs=True,
+        slurm_user=slurm_user,
+        user_cache_dir=user_cache_dir,
+        working_dir=workflow_dir,
+        working_dir_user=workflow_dir_user,
+        common_script_lines=worker_init,
+        slurm_account=slurm_account,
+    ) as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir_user,
+            logger_name=logger_name,
+            submit_setup_call=_slurm_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir: Path,
+    workflow_dir_user: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Process workflow (SLURM backend public interface)
+
+    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    """
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+        user_cache_dir=user_cache_dir,
+        slurm_user=slurm_user,
+        slurm_account=slurm_account,
+        worker_init=worker_init,
+    )
+    return new_dataset_attributes

fractal_server-2.0.0a2/fractal_server/app/runner/v2/deduplicate_list.py

@@ -0,0 +1,22 @@
+from typing import TypeVar
+
+from ....images import SingleImage
+from .task_interface import InitArgsModel
+
+T = TypeVar("T", SingleImage, InitArgsModel)
+
+
+def deduplicate_list(
+    this_list: list[T],
+) -> list[T]:
+    """
+    Custom replacement for `set(this_list)`, when items are non-hashable.
+    """
+    new_list_dict = []
+    new_list_objs = []
+    for this_obj in this_list:
+        this_dict = this_obj.dict()
+        if this_dict not in new_list_dict:
+            new_list_dict.append(this_dict)
+            new_list_objs.append(this_obj)
+    return new_list_objs

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v2/merge_outputs.py

@@ -2,7 +2,6 @@ from copy import copy
 
 from fractal_server.app.runner.v2.deduplicate_list import deduplicate_list
 from fractal_server.app.runner.v2.task_interface import TaskOutput
-from fractal_server.images import SingleImage
 
 
 def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
@@ -24,9 +23,7 @@ def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
             raise ValueError(f"{current_new_filters=} but {last_new_filters=}")
         last_new_filters = copy(current_new_filters)
 
-    final_image_list_updates = deduplicate_list(
-        final_image_list_updates, PydanticModel=SingleImage
-    )
+    final_image_list_updates = deduplicate_list(final_image_list_updates)
 
     additional_args = {}
     if last_new_filters is not None:

{fractal_server-2.0.0a0 → fractal_server-2.0.0a2}/fractal_server/app/runner/v2/runner.py

@@ -8,7 +8,7 @@ from typing import Optional
 
 from ....images import Filters
 from ....images import SingleImage
-from ....images.tools import _filter_image_list
+from ....images.tools import filter_image_list
 from ....images.tools import find_image_by_path
 from ....images.tools import match_filter
 from ..filenames import FILTERS_FILENAME
@@ -52,16 +52,15 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Get filtered images
-        pre_type_filters = copy(tmp_filters["types"])
-        pre_type_filters.update(wftask.input_filters["types"])
-        pre_attribute_filters = copy(tmp_filters["attributes"])
-        pre_attribute_filters.update(wftask.input_filters["attributes"])
-        filtered_images = _filter_image_list(
+        pre_filters = dict(
+            types=copy(tmp_filters["types"]),
+            attributes=copy(tmp_filters["attributes"]),
+        )
+        pre_filters["types"].update(wftask.input_filters["types"])
+        pre_filters["attributes"].update(wftask.input_filters["attributes"])
+        filtered_images = filter_image_list(
             images=tmp_images,
-            filters=Filters(
-                types=pre_type_filters,
-                attributes=pre_attribute_filters,
-            ),
+            filters=Filters(**pre_filters),
         )
         # Verify that filtered images comply with task input_types
         for image in filtered_images:
@@ -183,22 +182,26 @@ def execute_tasks_v2(
                 updated_attributes.update(image["attributes"])
                 updated_types.update(image["types"])
                 updated_types.update(task.output_types)
-                new_image = SingleImage(
+                new_image = dict(
                     path=image["path"],
                     origin=image["origin"],
                     attributes=updated_attributes,
                     types=updated_types,
                 )
+                # Validate new image
+                SingleImage(**new_image)
                 # Add image into the dataset image list
-                tmp_images.append(new_image.dict())
+                tmp_images.append(new_image)
 
         # Remove images from tmp_images
-        for image in current_task_output.image_list_removals:
+        for image_path in current_task_output.image_list_removals:
             image_search = find_image_by_path(
-                images=tmp_images, path=image["path"]
+                images=tmp_images, path=image_path
             )
-            if image_search["index"] is None:
-                raise
+            if image_search is None:
+                raise ValueError(
+                    f"Cannot remove missing image with path {image_path=}"
+                )
             else:
                 tmp_images.pop(image_search["index"])