fractal-server 2.0.0a0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.0.0a0"
+__VERSION__ = "2.0.0a1"

fractal_server/app/models/__init__.py

@@ -1,7 +1,6 @@
 """
 `models` module
 """
-from ..schemas.v1 import *  # noqa F401, F403  # FIXME: remove this
 from .security import *  # noqa: F401, F403
 from .state import State  # noqa: F401
 from .v1 import *  # noqa: F401, F403

fractal_server/app/models/v1/__init__.py

@@ -4,6 +4,7 @@
 from .dataset import Dataset  # noqa: F401
 from .dataset import Resource  # noqa: F401
 from .job import ApplyWorkflow  # noqa: F403, F401
+from .job import JobStatusTypeV1  # noqa: F401, F403
 from .project import Project  # noqa: F403, F401
 from .task import Task  # noqa: F403, F401
 from .workflow import Workflow  # noqa: F401, F403

fractal_server/app/routes/admin/v2.py

@@ -268,7 +268,6 @@ async def flag_task_v1_as_v2_compatible(
         )
 
     task.is_v2_compatible = is_v2_compatible
-    db.add(task)
     await db.commit()
     await db.close()
 

fractal_server/app/routes/api/v1/job.py

@@ -10,9 +10,9 @@ from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
-from ....models import ApplyWorkflow
-from ....models import JobStatusTypeV1
-from ....models import Project
+from ....models.v1 import ApplyWorkflow
+from ....models.v1 import JobStatusTypeV1
+from ....models.v1 import Project
 from ....runner.filenames import WORKFLOW_LOG_FILENAME
 from ....schemas.v1 import ApplyWorkflowReadV1
 from ....security import current_active_user

fractal_server/app/routes/api/v2/__init__.py

@@ -3,11 +3,11 @@
 """
 from fastapi import APIRouter
 
-from .apply import router as runner_router_v2
 from .dataset import router as dataset_router_v2
 from .images import router as images_routes_v2
 from .job import router as job_router_v2
 from .project import router as project_router_v2
+from .submit import router as submit_job_router_v2
 from .task import router as task_router_v2
 from .task_collection import router as task_collection_router_v2
 from .workflow import router as workflow_router_v2
@@ -19,7 +19,7 @@ router_api_v2.include_router(dataset_router_v2, tags=["V2 Dataset"])
 router_api_v2.include_router(job_router_v2, tags=["V2 Job"])
 router_api_v2.include_router(images_routes_v2, tags=["V2 Images"])
 router_api_v2.include_router(project_router_v2, tags=["V2 Project"])
-router_api_v2.include_router(runner_router_v2, tags=["V2 Apply-workflow"])
+router_api_v2.include_router(submit_job_router_v2, tags=["V2 Submit Job"])
 router_api_v2.include_router(task_router_v2, prefix="/task", tags=["V2 Task"])
 router_api_v2.include_router(
     task_collection_router_v2, prefix="/task", tags=["V2 Task Collection"]

fractal_server/app/routes/api/v2/dataset.py

@@ -186,7 +186,6 @@ async def delete_dataset(
     jobs = res.scalars().all()
     for job in jobs:
         job.dataset_id = None
-        await db.merge(job)
     await db.commit()
 
     # Delete dataset

fractal_server/app/routes/api/v2/images.py

@@ -17,7 +17,7 @@ from fractal_server.app.security import current_active_user
 from fractal_server.app.security import User
 from fractal_server.images import Filters
 from fractal_server.images import SingleImage
-from fractal_server.images.tools import match_filter_SingleImage
+from fractal_server.images.tools import match_filter
 
 router = APIRouter()
 
@@ -68,7 +68,6 @@ async def post_new_image(
     dataset.images.append(new_image.dict())
     flag_modified(dataset, "images")
 
-    await db.merge(dataset)
     await db.commit()
 
     return Response(status_code=status.HTTP_201_CREATED)
@@ -106,9 +105,7 @@ async def query_dataset_images(
         images = [
             image
             for image in images
-            if match_filter_SingleImage(
-                SingleImage(**image), Filters(**dataset.filters)
-            )
+            if match_filter(image, Filters(**dataset.filters))
         ]
 
     attributes = {}
@@ -138,8 +135,8 @@ async def query_dataset_images(
             images = [
                 image
                 for image in images
-                if match_filter_SingleImage(
-                    SingleImage(**image),
+                if match_filter(
+                    image,
                     Filters(**query.filters.dict()),
                 )
             ]
@@ -159,7 +156,6 @@ async def query_dataset_images(
 
     if total_count == 0:
         page = 1
-        page_size = 0
     else:
         last_page = (total_count // page_size) + (total_count % page_size > 0)
         if page > last_page:
@@ -206,7 +202,6 @@ async def delete_dataset_images(
     dataset.images.remove(image_to_remove)
     flag_modified(dataset, "images")
 
-    await db.merge(dataset)
     await db.commit()
 
     return Response(status_code=status.HTTP_204_NO_CONTENT)

fractal_server/app/routes/api/v2/project.py

@@ -167,7 +167,6 @@ async def delete_project(
         jobs = res.scalars().all()
         for job in jobs:
             job.workflow_id = None
-            await db.merge(job)
         # Delete workflow
         await db.delete(wf)
     await db.commit()
@@ -184,7 +183,6 @@ async def delete_project(
         jobs = res.scalars().all()
         for job in jobs:
             job.dataset_id = None
-            await db.merge(job)
         # Delete dataset
         await db.delete(ds)
     await db.commit()
@@ -195,7 +193,6 @@ async def delete_project(
     jobs = res.scalars().all()
     for job in jobs:
         job.project_id = None
-        await db.merge(job)
 
     await db.commit()
 

fractal_server/app/routes/api/v2/{apply.py → submit.py}

@@ -36,7 +36,7 @@ router = APIRouter()
 
 
 @router.post(
-    "/project/{project_id}/workflow/{workflow_id}/apply/",
+    "/project/{project_id}/job/submit/",
     status_code=status.HTTP_202_ACCEPTED,
     response_model=JobReadV2,
 )

fractal_server/app/routes/api/v2/workflow.py

@@ -213,7 +213,6 @@ async def delete_workflow(
     jobs = res.scalars().all()
     for job in jobs:
         job.workflow_id = None
-        await db.merge(job)
     await db.commit()
 
     # Delete workflow

fractal_server/app/runner/executors/slurm/executor.py

@@ -46,6 +46,7 @@ from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
+from fractal_server.app.runner.components import _COMPONENT_KEY_
 
 
 logger = set_logger(__name__)
@@ -544,7 +545,7 @@ class FractalSlurmExecutor(SlurmExecutor):
         single_task_submission: bool = False,
         args: Optional[Sequence[Any]] = None,
         kwargs: Optional[dict] = None,
-        components: list[Any] = None,
+        components: Optional[list[Any]] = None,
     ) -> Future:
         """
         Submit a multi-task job to the pool, where each task is handled via the
@@ -580,6 +581,10 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         # Define slurm-job-related files
         if single_task_submission:
+            if components is not None:
+                raise ValueError(
+                    f"{single_task_submission=} but components is not None"
+                )
             job = SlurmJob(
                 slurm_file_prefix=slurm_file_prefix,
                 num_tasks_tot=1,
@@ -603,15 +608,23 @@ class FractalSlurmExecutor(SlurmExecutor):
                 num_tasks_tot=num_tasks_tot,
                 slurm_config=slurm_config,
             )
-            job.wftask_file_prefixes = tuple(
-                get_task_file_paths(
-                    workflow_dir=task_files.workflow_dir,
-                    workflow_dir_user=task_files.workflow_dir_user,
-                    task_order=task_files.task_order,
-                    component=component,
-                ).file_prefix
-                for component in components
-            )
+
+            _prefixes = []
+            for component in components:
+                if isinstance(component, dict):
+                    # This is needed for V2
+                    actual_component = component.get(_COMPONENT_KEY_, None)
+                else:
+                    actual_component = component
+                _prefixes.append(
+                    get_task_file_paths(
+                        workflow_dir=task_files.workflow_dir,
+                        workflow_dir_user=task_files.workflow_dir_user,
+                        task_order=task_files.task_order,
+                        component=actual_component,
+                    ).file_prefix
+                )
+            job.wftask_file_prefixes = tuple(_prefixes)
 
         # Define I/O pickle file names/paths
         job.input_pickle_files = tuple(

fractal_server/app/runner/task_files.py

@@ -1,4 +1,3 @@
-from functools import lru_cache
 from pathlib import Path
 from typing import Optional
 
@@ -84,7 +83,6 @@ class TaskFiles:
         )
 
 
-@lru_cache()
 def get_task_file_paths(
     workflow_dir: Path,
     workflow_dir_user: Path,

fractal_server/app/runner/v1/__init__.py

@@ -34,8 +34,8 @@ from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from ..filenames import WORKFLOW_LOG_FILENAME
-from ..v1._local import process_workflow as local_process_workflow
-from ..v1._slurm import process_workflow as slurm_process_workflow
+from ._local import process_workflow as local_process_workflow
+from ._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
 from .common import validate_workflow_compatibility  # noqa: F401
 from .handle_failed_job import assemble_history_failed_job

fractal_server/app/runner/v1/_local/__init__.py

@@ -25,11 +25,11 @@ from typing import Optional
 
 from ....models import Workflow  # FIXME: this is v1 specific
 from ...async_wrap import async_wrap
-from ...executors.local.executor import FractalThreadPoolExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks  # FIXME: this is v1 specific
 from ..common import TaskParameters  # FIXME: this is v1 specific
 from ._submit_setup import _local_submit_setup
+from .executor import FractalThreadPoolExecutor
 
 
 def _process_workflow(

fractal_server/app/runner/{executors/local → v1/_local}/executor.py

@@ -18,8 +18,8 @@ from typing import Iterable
 from typing import Optional
 from typing import Sequence
 
-from ...v1._local._local_config import get_default_local_backend_config
-from ...v1._local._local_config import LocalBackendConfig
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
 
 
 class FractalThreadPoolExecutor(ThreadPoolExecutor):

fractal_server/app/runner/v2/__init__.py

@@ -31,7 +31,6 @@ from ._slurm import process_workflow as slurm_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_images_failed_job
-from .runner import execute_tasks_v2  # noqa
 from fractal_server import __VERSION__
 
 _backends = {}

fractal_server/app/runner/v2/_local/__init__.py

@@ -25,12 +25,10 @@ from typing import Optional
 from ....models.v2 import DatasetV2
 from ....models.v2 import WorkflowV2
 from ...async_wrap import async_wrap
-from ...executors.local.executor import FractalThreadPoolExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
-
-# from typing import Any
+from .executor import FractalThreadPoolExecutor
 
 
 def _process_workflow(

fractal_server/app/runner/v2/_local/executor.py

@@ -0,0 +1,100 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Custom version of Python
+[ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor)).
+"""
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+from typing import Iterable
+from typing import Optional
+from typing import Sequence
+
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
+
+
+class FractalThreadPoolExecutor(ThreadPoolExecutor):
+    """
+    Custom version of
+    [ThreadPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor))
+    that overrides the `submit` and `map` methods
+    """
+
+    def submit(
+        self,
+        *args,
+        local_backend_config: Optional[LocalBackendConfig] = None,
+        **kwargs,
+    ):
+        """
+        Compared to the `ThreadPoolExecutor` method, here we accept an addition
+        keyword argument (`local_backend_config`), which is then simply
+        ignored.
+        """
+        return super().submit(*args, **kwargs)
+
+    def map(
+        self,
+        fn: Callable,
+        *iterables: Sequence[Iterable],
+        local_backend_config: Optional[LocalBackendConfig] = None,
+    ):
+        """
+        Custom version of the `Executor.map` method
+
+        The main change with the respect to the original `map` method is that
+        the list of tasks to be executed is split into chunks, and then
+        `super().map` is called (sequentially) on each chunk. The goal of this
+        change is to limit parallelism, e.g. due to limited computational
+        resources.
+
+        Other changes from the `concurrent.futures` `map` method:
+
+        1. Removed `timeout` argument;
+        2. Removed `chunksize`;
+        3. All iterators (both inputs and output ones) are transformed into
+           lists.
+
+        Args:
+            fn: A callable function.
+            iterables: The argument iterables (one iterable per argument of
+                       `fn`).
+           local_backend_config: The backend configuration, needed to extract
+                                 `parallel_tasks_per_job`.
+        """
+
+        # Preliminary check
+        iterable_lengths = [len(it) for it in iterables]
+        if not len(set(iterable_lengths)) == 1:
+            raise ValueError("Iterables have different lengths.")
+
+        # Set total number of arguments
+        n_elements = len(iterables[0])
+
+        # Set parallel_tasks_per_job
+        if local_backend_config is None:
+            local_backend_config = get_default_local_backend_config()
+        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
+        if parallel_tasks_per_job is None:
+            parallel_tasks_per_job = n_elements
+
+        # Execute tasks, in chunks of size parallel_tasks_per_job
+        results = []
+        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+            chunk_iterables = [
+                it[ind_chunk : ind_chunk + parallel_tasks_per_job]  # noqa
+                for it in iterables
+            ]
+            map_iter = super().map(fn, *chunk_iterables)
+            results.extend(list(map_iter))
+
+        return iter(results)

fractal_server/app/runner/v2/_slurm/__init__.py

@@ -21,11 +21,13 @@ from typing import Any
 from typing import Optional
 from typing import Union
 
-from fractal_server.app.models.v2 import WorkflowV2
-
-# from ...executors.slurm.executor import FractalSlurmExecutor
-# from ._submit_setup import _slurm_submit_setup
-# from .get_slurm_config import get_slurm_config
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...executors.slurm.executor import FractalSlurmExecutor
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _slurm_submit_setup
 
 # from .._common import execute_tasks
 # from ..common import async_wrap
@@ -36,10 +38,7 @@ from fractal_server.app.models.v2 import WorkflowV2
 def _process_workflow(
     *,
     workflow: WorkflowV2,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
+    dataset: DatasetV2,
     logger_name: str,
     workflow_dir: Path,
     workflow_dir_user: Path,
@@ -55,103 +54,86 @@ def _process_workflow(
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
-    workflow tasks and returns the output dataset metadata.
+    workflow tasks and returns the new dataset attributes
 
     Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
 
     Returns:
-        output_dataset_metadata: Metadata of the output dataset
+        new_dataset_attributes:
     """
 
-    raise NotImplementedError
-
-    # if not slurm_user:
-    #     raise RuntimeError(
-    #         "slurm_user argument is required, for slurm backend"
-    #     )
-
-    # if isinstance(worker_init, str):
-    #     worker_init = worker_init.split("\n")
-
-    # with FractalSlurmExecutor(
-    #     debug=True,
-    #     keep_logs=True,
-    #     slurm_user=slurm_user,
-    #     user_cache_dir=user_cache_dir,
-    #     working_dir=workflow_dir,
-    #     working_dir_user=workflow_dir_user,
-    #     common_script_lines=worker_init,
-    #     slurm_account=slurm_account,
-    # ) as executor:
-    #     output_task_pars = execute_tasks(
-    #         executor=executor,
-    #         task_list=workflow.task_list[
-    #             first_task_index : (last_task_index + 1)  # noqa
-    #         ],  # noqa
-    #         task_pars=TaskParameters(
-    #             input_paths=input_paths,
-    #             output_path=output_path,
-    #             metadata=input_metadata,
-    #             history=input_history,
-    #         ),
-    #         workflow_dir=workflow_dir,
-    #         workflow_dir_user=workflow_dir_user,
-    #         submit_setup_call=_slurm_submit_setup,
-    #         logger_name=logger_name,
-    #     )
-    # output_dataset_metadata_history = dict(
-    #     metadata=output_task_pars.metadata, history=output_task_pars.history
-    # )
-    # return output_dataset_metadata_history
+    if not slurm_user:
+        raise RuntimeError(
+            "slurm_user argument is required, for slurm backend"
+        )
+
+    if isinstance(worker_init, str):
+        worker_init = worker_init.split("\n")
+
+    with FractalSlurmExecutor(
+        debug=True,
+        keep_logs=True,
+        slurm_user=slurm_user,
+        user_cache_dir=user_cache_dir,
+        working_dir=workflow_dir,
+        working_dir_user=workflow_dir_user,
+        common_script_lines=worker_init,
+        slurm_account=slurm_account,
+    ) as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir=workflow_dir,
+            workflow_dir_user=workflow_dir_user,
+            logger_name=logger_name,
+            submit_setup_call=_slurm_submit_setup,
+        )
+    return new_dataset_attributes
 
 
 async def process_workflow(
     *,
     workflow: WorkflowV2,
-    input_paths: list[Path],
-    output_path: Path,
-    input_metadata: dict[str, Any],
-    input_history: list[dict[str, Any]],
-    logger_name: str,
+    dataset: DatasetV2,
     workflow_dir: Path,
     workflow_dir_user: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-    first_task_index: Optional[int] = None,
-    last_task_index: Optional[int] = None,
-) -> dict[str, Any]:
+) -> dict:
     """
     Process workflow (SLURM backend public interface)
 
     Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
     """
 
-    raise NotImplementedError
-
-    # # Set values of first_task_index and last_task_index
-    # num_tasks = len(workflow.task_list)
-    # first_task_index, last_task_index = set_start_and_last_task_index(
-    #     num_tasks,
-    #     first_task_index=first_task_index,
-    #     last_task_index=last_task_index,
-    # )
-
-    # output_dataset_metadata_history = await async_wrap(_process_workflow)(
-    #     workflow=workflow,
-    #     input_paths=input_paths,
-    #     output_path=output_path,
-    #     input_metadata=input_metadata,
-    #     input_history=input_history,
-    #     logger_name=logger_name,
-    #     workflow_dir=workflow_dir,
-    #     workflow_dir_user=workflow_dir_user,
-    #     slurm_user=slurm_user,
-    #     slurm_account=slurm_account,
-    #     user_cache_dir=user_cache_dir,
-    #     worker_init=worker_init,
-    #     first_task_index=first_task_index,
-    #     last_task_index=last_task_index,
-    # )
-    # return output_dataset_metadata_history
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir=workflow_dir,
+        workflow_dir_user=workflow_dir_user,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+        user_cache_dir=user_cache_dir,
+        slurm_user=slurm_user,
+        slurm_account=slurm_account,
+        worker_init=worker_init,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/deduplicate_list.py

@@ -1,7 +1,5 @@
 from typing import TypeVar
 
-from pydantic.main import ModelMetaclass
-
 from ....images import SingleImage
 from .task_interface import InitArgsModel
 
@@ -9,16 +7,16 @@ T = TypeVar("T", SingleImage, InitArgsModel)
 
 
 def deduplicate_list(
-    this_list: list[T], PydanticModel: ModelMetaclass
+    this_list: list[T],
 ) -> list[T]:
     """
-    Custom replacement for `set(this_list)`, when items are Pydantic-model
-    instances and then non-hashable (e.g. SingleImage or InitArgsModel).
+    Custom replacement for `set(this_list)`, when items are non-hashable.
     """
-    this_list_dict = [this_item.dict() for this_item in this_list]
     new_list_dict = []
-    for this_dict in this_list_dict:
+    new_list_objs = []
+    for this_obj in this_list:
+        this_dict = this_obj.dict()
         if this_dict not in new_list_dict:
             new_list_dict.append(this_dict)
-    new_list = [PydanticModel(**this_dict) for this_dict in new_list_dict]
-    return new_list
+            new_list_objs.append(this_obj)
+    return new_list_objs

fractal_server/app/runner/v2/merge_outputs.py

@@ -2,7 +2,6 @@ from copy import copy
 
 from fractal_server.app.runner.v2.deduplicate_list import deduplicate_list
 from fractal_server.app.runner.v2.task_interface import TaskOutput
-from fractal_server.images import SingleImage
 
 
 def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
@@ -24,9 +23,7 @@ def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
             raise ValueError(f"{current_new_filters=} but {last_new_filters=}")
         last_new_filters = copy(current_new_filters)
 
-    final_image_list_updates = deduplicate_list(
-        final_image_list_updates, PydanticModel=SingleImage
-    )
+    final_image_list_updates = deduplicate_list(final_image_list_updates)
 
     additional_args = {}
     if last_new_filters is not None: