fractal-server 2.13.1__py3-none-any.whl → 2.14.0a0__py3-none-any.whl

fractal_server/app/routes/api/v2/workflow.py

@@ -7,10 +7,13 @@ from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
+from sqlmodel import delete
 from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
+from ....models.v2 import HistoryItemV2
+from ....models.v2 import ImageStatus
 from ....models.v2 import JobV2
 from ....models.v2 import ProjectV2
 from ....models.v2 import WorkflowV2
@@ -225,14 +228,26 @@ async def delete_workflow(
             ),
         )
 
-    # Cascade operations: set foreign-keys to null for jobs which are in
-    # relationship with the current workflow
+    # Cascade operation: set foreign-keys to null for jobs and history items
+    # which are in relationship with the current workflow.
     stm = select(JobV2).where(JobV2.workflow_id == workflow_id)
     res = await db.execute(stm)
     jobs = res.scalars().all()
     for job in jobs:
         job.workflow_id = None
 
+    wft_ids = [wft.id for wft in workflow.task_list]
+    stm = select(HistoryItemV2).where(
+        HistoryItemV2.workflowtask_id.in_(wft_ids)
+    )
+    res = await db.execute(stm)
+    history_items = res.scalars().all()
+    for history_item in history_items:
+        history_item.workflowtask_id = None
+
+    stm = delete(ImageStatus).where(ImageStatus.workflowtask_id.in_(wft_ids))
+    await db.execute(stm)
+
     # Delete workflow
     await db.delete(workflow)
     await db.commit()
@@ -244,7 +259,7 @@ async def delete_workflow(
     "/project/{project_id}/workflow/{workflow_id}/export/",
     response_model=WorkflowExportV2,
 )
-async def export_worfklow(
+async def export_workflow(
     project_id: int,
     workflow_id: int,
     user: UserOAuth = Depends(current_active_user),

fractal_server/app/routes/api/v2/workflowtask.py

@@ -6,6 +6,8 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import delete
+from sqlmodel import select
 
 from ....db import AsyncSession
 from ....db import get_async_db
@@ -15,6 +17,8 @@ from ._aux_functions import _workflow_insert_task
 from ._aux_functions_tasks import _check_type_filters_compatibility
 from ._aux_functions_tasks import _get_task_read_access
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import HistoryItemV2
+from fractal_server.app.models.v2 import ImageStatus
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.schemas.v2 import WorkflowTaskCreateV2
@@ -333,6 +337,24 @@ async def delete_workflowtask(
         db=db,
     )
 
+    # Cascade operations:
+    # * set foreign-keys to null for history items which are in relationship
+    #   with the current workflowtask;
+    # * delete ImageStatus in relationship with the current workflowtask.
+    stm = select(HistoryItemV2).where(
+        HistoryItemV2.workflowtask_id == db_workflow_task.id
+    )
+    res = await db.execute(stm)
+    history_items = res.scalars().all()
+    for history_item in history_items:
+        history_item.workflowtask_id = None
+
+    stm = delete(ImageStatus).where(
+        ImageStatus.workflowtask_id == db_workflow_task.id
+    )
+    await db.execute(stm)
+
+    # Delete WorkflowTask
     await db.delete(db_workflow_task)
     await db.commit()
 

fractal_server/app/runner/executors/base_runner.py

@@ -0,0 +1,114 @@
+from typing import Any
+
+from fractal_server.app.runner.components import _COMPONENT_KEY_
+
+
+class BaseRunner(object):
+    """
+    Base class for Fractal runners.
+    """
+
+    def shutdown(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def submit(
+        self,
+        func: callable,
+        parameters: dict[str, Any],
+        history_item_id: int,
+        in_compound_task: bool,
+        **kwargs,
+    ) -> tuple[Any, BaseException]:
+        """
+        Run a single fractal task.
+
+        # FIXME: Describe more in detail
+
+        Args:
+            func:
+                Function to be executed.
+            parameters:
+                Dictionary of parameters. Must include `zarr_urls` key.
+            history_item_id:
+                Database ID of the corresponding `HistoryItemV2` entry.
+            in_compound_task:
+                Whether this is the init part of a compound task.
+            kwargs:
+                Runner-specific parameters.
+        """
+        raise NotImplementedError()
+
+    def multisubmit(
+        self,
+        func: callable,
+        list_parameters: list[dict[str, Any]],
+        history_item_id: int,
+        in_compound_task: bool,
+        **kwargs,
+    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
+        """
+        Run a parallel fractal task.
+
+        # FIXME: Describe more in detail
+
+        Args:
+            func:
+                Function to be executed.
+            list_parameters:
+                List of dictionaries of parameters. Each one must include a
+                `zarr_url` key.
+            history_item_id:
+                Database ID of the corresponding `HistoryItemV2` entry.
+            in_compound_task:
+                Whether this is the compute part of a compound task.
+            kwargs:
+                Runner-specific parameters.
+        """
+        raise NotImplementedError()
+
+    def validate_submit_parameters(self, parameters: dict[str, Any]) -> None:
+        """
+        Validate parameters for `submit` method
+
+        Args:
+            parameters: Parameters dictionary.
+        """
+        if not isinstance(parameters, dict):
+            raise ValueError("`parameters` must be a dictionary.")
+        if "zarr_urls" not in parameters.keys():
+            raise ValueError(
+                f"No 'zarr_urls' key in in {list(parameters.keys())}"
+            )
+        if _COMPONENT_KEY_ not in parameters.keys():
+            raise ValueError(
+                f"No '{_COMPONENT_KEY_}' key in in {list(parameters.keys())}"
+            )
+
+    def validate_multisubmit_parameters(
+        self,
+        list_parameters: list[dict[str, Any]],
+        in_compound_task: bool,
+    ) -> None:
+        """
+        Validate parameters for `multi_submit` method
+
+        Args:
+            list_parameters: List of parameters dictionaries.
+            in_compound_task:
+               Whether this is the compute part of a compound task.
+        """
+        for single_kwargs in list_parameters:
+            if not isinstance(single_kwargs, dict):
+                raise RuntimeError("kwargs itemt must be a dictionary.")
+            if "zarr_url" not in single_kwargs.keys():
+                raise RuntimeError(
+                    f"No 'zarr_url' key in in {list(single_kwargs.keys())}"
+                )
+            if _COMPONENT_KEY_ not in single_kwargs.keys():
+                raise ValueError(
+                    f"No '{_COMPONENT_KEY_}' key in in {list(single_kwargs.keys())}"
+                )
+        if not in_compound_task:
+            zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
+            if len(zarr_urls) != len(set(zarr_urls)):
+                raise RuntimeError("Non-unique zarr_urls")

fractal_server/app/runner/{v2/_local → executors/local}/_local_config.py

@@ -20,9 +20,9 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import ValidationError
 
-from .....config import get_settings
-from .....syringe import Inject
-from ....models.v2 import WorkflowTaskV2
+from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.config import get_settings
+from fractal_server.syringe import Inject
 
 
 class LocalBackendConfigError(ValueError):

fractal_server/app/runner/executors/local/_submit_setup.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+from typing import Any
+from typing import Literal
+
+from ...task_files import TaskFiles
+from ._local_config import get_local_backend_config
+from fractal_server.app.models.v2 import WorkflowTaskV2
+
+
+def _local_submit_setup(
+    *,
+    wftask: WorkflowTaskV2,
+    root_dir_local: Path,
+    root_dir_remote: Path,
+    which_type: Literal["non_parallel", "parallel"],
+) -> dict[str, Any]:
+    """
+    Collect WorkflowTask-specific configuration parameters from different
+    sources, and inject them for execution.
+
+    FIXME
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the configuration is to be assembled
+        workflow_dir_local:
+            Not used in this function.
+        workflow_dir_remote:
+            Not used in this function.
+
+    Returns:
+        submit_setup_dict:
+            A dictionary that will be passed on to
+            `FractalThreadPoolExecutor.submit` and
+            `FractalThreadPoolExecutor.map`, so as to set extra options.
+    """
+
+    local_backend_config = get_local_backend_config(
+        wftask=wftask,
+        which_type=which_type,
+    )
+
+    # Get TaskFiles object
+    task_files = TaskFiles(
+        root_dir_local=root_dir_local,
+        root_dir_remote=root_dir_remote,
+        task_order=wftask.order,
+        task_name=wftask.task.name,
+    )
+
+    return dict(
+        local_backend_config=local_backend_config,
+        task_files=task_files,
+    )

fractal_server/app/runner/executors/local/runner.py

@@ -0,0 +1,200 @@
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+from typing import Optional
+
+from ._local_config import get_default_local_backend_config
+from ._local_config import LocalBackendConfig
+from fractal_server.app.history import HistoryItemImageStatus
+from fractal_server.app.history import update_all_images
+from fractal_server.app.history import update_single_image
+from fractal_server.app.history import update_single_image_logfile
+from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.runner.executors.base_runner import BaseRunner
+from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.logger import set_logger
+
+logger = set_logger(__name__)
+
+
+class LocalRunner(BaseRunner):
+    executor: ThreadPoolExecutor
+    root_dir_local: Path
+
+    def __init__(
+        self,
+        root_dir_local: Path,
+    ):
+
+        self.root_dir_local = root_dir_local
+        self.root_dir_local.mkdir(parents=True, exist_ok=True)
+        self.executor = ThreadPoolExecutor()
+        logger.debug("Create LocalRunner")
+
+    def __enter__(self):
+        logger.debug("Enter LocalRunner")
+        return self
+
+    def shutdown(self):
+        logger.debug("Now shut LocalRunner.executor down")
+        self.executor.shutdown(
+            wait=False,
+            cancel_futures=True,
+        )
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        logger.debug("Exit LocalRunner")
+        self.shutdown()
+        return self.executor.__exit__(exc_type, exc_val, exc_tb)
+
+    def submit(
+        self,
+        func: callable,
+        parameters: dict[str, Any],
+        history_item_id: int,
+        task_files: TaskFiles,
+        in_compound_task: bool = False,
+        **kwargs,
+    ) -> tuple[Any, Exception]:
+        logger.debug("[submit] START")
+
+        current_task_files = TaskFiles(
+            **task_files.model_dump(
+                exclude={"component"},
+            ),
+            component=parameters[_COMPONENT_KEY_],
+        )
+
+        self.validate_submit_parameters(parameters)
+        workdir_local = current_task_files.wftask_subfolder_local
+        workdir_local.mkdir()
+        # SUBMISSION PHASE
+        future = self.executor.submit(func, parameters=parameters)
+
+        # RETRIEVAL PHASE
+        try:
+            result = future.result()
+            if not in_compound_task:
+                update_all_images(
+                    history_item_id=history_item_id,
+                    status=HistoryItemImageStatus.DONE,
+                    logfile=current_task_files.log_file_local,
+                )
+            logger.debug(f"[submit] END {result=}")
+            return result, None
+        except Exception as e:
+            exception = e
+            update_all_images(
+                history_item_id=history_item_id,
+                status=HistoryItemImageStatus.FAILED,
+                logfile=current_task_files.log_file_local,
+            )
+            logger.debug(f"[submit] END {exception=}")
+            return None, exception
+
+    def multisubmit(
+        self,
+        func: callable,
+        list_parameters: list[dict],
+        history_item_id: int,
+        task_files: TaskFiles,
+        in_compound_task: bool = False,
+        local_backend_config: Optional[LocalBackendConfig] = None,
+        **kwargs,
+    ):
+        logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
+
+        self.validate_multisubmit_parameters(
+            list_parameters=list_parameters,
+            in_compound_task=in_compound_task,
+        )
+
+        workdir_local = task_files.wftask_subfolder_local
+        if not in_compound_task:
+            workdir_local.mkdir()
+
+        # Get local_backend_config
+        if local_backend_config is None:
+            local_backend_config = get_default_local_backend_config()
+
+        # Set `n_elements` and `parallel_tasks_per_job`
+        n_elements = len(list_parameters)
+        parallel_tasks_per_job = local_backend_config.parallel_tasks_per_job
+        if parallel_tasks_per_job is None:
+            parallel_tasks_per_job = n_elements
+
+        original_task_files = task_files
+
+        # Execute tasks, in chunks of size `parallel_tasks_per_job`
+        results = {}
+        exceptions = {}
+        for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+            list_parameters_chunk = list_parameters[
+                ind_chunk : ind_chunk + parallel_tasks_per_job
+            ]
+            from concurrent.futures import Future
+
+            active_futures: dict[int, Future] = {}
+            active_task_files: dict[int, TaskFiles] = {}
+            for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
+                positional_index = ind_chunk + ind_within_chunk
+                component = kwargs[_COMPONENT_KEY_]
+                future = self.executor.submit(func, parameters=kwargs)
+                active_futures[positional_index] = future
+                active_task_files[positional_index] = TaskFiles(
+                    **original_task_files.model_dump(exclude={"component"}),
+                    component=component,
+                )
+
+            while active_futures:
+                # FIXME: add shutdown detection
+                # if file exists: cancel all futures, and raise
+                finished_futures = [
+                    keyval
+                    for keyval in active_futures.items()
+                    if not keyval[1].running()
+                ]
+                for positional_index, fut in finished_futures:
+                    active_futures.pop(positional_index)
+                    current_task_files = active_task_files.pop(
+                        positional_index
+                    )
+                    zarr_url = list_parameters[positional_index]["zarr_url"]
+                    if not in_compound_task:
+                        update_single_image_logfile(
+                            history_item_id=history_item_id,
+                            zarr_url=zarr_url,
+                            logfile=current_task_files.log_file_local,
+                        )
+                    try:
+                        results[positional_index] = fut.result()
+                        print(f"Mark {zarr_url=} as done, {kwargs}")
+                        if not in_compound_task:
+                            update_single_image(
+                                history_item_id=history_item_id,
+                                zarr_url=zarr_url,
+                                status=HistoryItemImageStatus.DONE,
+                            )
+                    except Exception as e:
+                        print(f"Mark {zarr_url=} as failed, {kwargs} - {e}")
+                        exceptions[positional_index] = e
+                        if not in_compound_task:
+                            update_single_image(
+                                history_item_id=history_item_id,
+                                zarr_url=zarr_url,
+                                status=HistoryItemImageStatus.FAILED,
+                            )
+        if in_compound_task:
+            if exceptions == {}:
+                update_all_images(
+                    history_item_id=history_item_id,
+                    status=HistoryItemImageStatus.DONE,
+                )
+            else:
+                update_all_images(
+                    history_item_id=history_item_id,
+                    status=HistoryItemImageStatus.FAILED,
+                )
+        logger.debug(f"[multisubmit] END, {results=}, {exceptions=}")
+
+        return results, exceptions

fractal_server/app/runner/executors/{slurm → slurm_common}/_batching.py

@@ -14,7 +14,7 @@ Submodule to determine the number of total/parallel tasks per SLURM job.
 import math
 from typing import Optional
 
-from .....logger import set_logger
+from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/executors/{slurm → slurm_common}/_slurm_config.py

@@ -22,9 +22,9 @@ from pydantic import ConfigDict
 from pydantic import Field
 from pydantic import ValidationError
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
+from fractal_server.config import get_settings
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/{v2/_slurm_ssh → executors/slurm_common}/_submit_setup.py

@@ -14,11 +14,12 @@ Submodule to define _slurm_submit_setup, which is also the reference
 implementation of `submit_setup_call`.
 """
 from pathlib import Path
+from typing import Any
 from typing import Literal
 
-from ...task_files import get_task_file_paths
+from ...task_files import TaskFiles
 from fractal_server.app.models.v2 import WorkflowTaskV2
-from fractal_server.app.runner.v2._slurm_common.get_slurm_config import (
+from fractal_server.app.runner.executors.slurm_common.get_slurm_config import (
     get_slurm_config,
 )
 
@@ -26,14 +27,16 @@ from fractal_server.app.runner.v2._slurm_common.get_slurm_config import (
 def _slurm_submit_setup(
     *,
     wftask: WorkflowTaskV2,
-    workflow_dir_local: Path,
-    workflow_dir_remote: Path,
+    root_dir_local: Path,
+    root_dir_remote: Path,
     which_type: Literal["non_parallel", "parallel"],
-) -> dict[str, object]:
+) -> dict[str, Any]:
     """
-    Collect WorfklowTask-specific configuration parameters from different
+    Collect WorkflowTask-specific configuration parameters from different
     sources, and inject them for execution.
 
+    FIXME
+
     Here goes all the logic for reading attributes from the appropriate sources
     and transforming them into an appropriate `SlurmConfig` object (encoding
     SLURM configuration) and `TaskFiles` object (with details e.g. about file
@@ -68,16 +71,14 @@ def _slurm_submit_setup(
     )
 
     # Get TaskFiles object
-    task_files = get_task_file_paths(
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
+    task_files = TaskFiles(
+        root_dir_local=root_dir_local,
+        root_dir_remote=root_dir_remote,
         task_order=wftask.order,
         task_name=wftask.task.name,
     )
 
-    # Prepare and return output dictionary
-    submit_setup_dict = dict(
+    return dict(
         slurm_config=slurm_config,
         task_files=task_files,
     )
-    return submit_setup_dict

fractal_server/app/runner/{v2/_slurm_common → executors/slurm_common}/get_slurm_config.py

@@ -2,18 +2,12 @@ from pathlib import Path
 from typing import Literal
 from typing import Optional
 
+from ._slurm_config import _parse_mem_value
+from ._slurm_config import load_slurm_config_file
+from ._slurm_config import logger
+from ._slurm_config import SlurmConfig
+from ._slurm_config import SlurmConfigError
 from fractal_server.app.models.v2 import WorkflowTaskV2
-from fractal_server.app.runner.executors.slurm._slurm_config import (
-    _parse_mem_value,
-)
-from fractal_server.app.runner.executors.slurm._slurm_config import (
-    load_slurm_config_file,
-)
-from fractal_server.app.runner.executors.slurm._slurm_config import logger
-from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
-from fractal_server.app.runner.executors.slurm._slurm_config import (
-    SlurmConfigError,
-)
 
 
 def get_slurm_config(
@@ -142,8 +136,8 @@ def get_slurm_config(
     extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
     if len(set(extra_lines)) != len(extra_lines):
         logger.debug(
-            "[get_slurm_config] Removing repeated elements "
-            f"from {extra_lines=}."
+            "[get_slurm_config] Removing repeated elements from "
+            f"{extra_lines=}."
         )
         extra_lines = list(set(extra_lines))
     slurm_dict["extra_lines"] = extra_lines
@@ -162,8 +156,8 @@ def get_slurm_config(
 
     # Put everything together
     logger.debug(
-        "[get_slurm_config] Now create a SlurmConfig object based "
-        f"on {slurm_dict=}"
+        "[get_slurm_config] Now create a SlurmConfig object based on "
+        f"{slurm_dict=}"
     )
     slurm_config = SlurmConfig(**slurm_dict)
 

fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_executor_wait_thread.py

@@ -4,8 +4,8 @@ import time
 import traceback
 from itertools import count
 
-from ......logger import set_logger
 from fractal_server.app.runner.exceptions import JobExecutionError
+from fractal_server.logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/_slurm_job.py

@@ -2,7 +2,7 @@ import uuid
 from pathlib import Path
 from typing import Optional
 
-from fractal_server.app.runner.executors.slurm._slurm_config import (
+from fractal_server.app.runner.executors.slurm_common._slurm_config import (
     SlurmConfig,
 )
 

fractal_server/app/runner/executors/{slurm/ssh → slurm_ssh}/executor.py

@@ -15,22 +15,21 @@ from typing import Sequence
 
 import cloudpickle
 
-from ....filenames import SHUTDOWN_FILENAME
-from ....task_files import get_task_file_paths
-from ....task_files import TaskFiles
-from ....versions import get_versions
-from ..._job_states import STATES_FINISHED
-from ...slurm._slurm_config import SlurmConfig
-from .._batching import heuristics
-from ..utils_executors import get_pickle_file_path
-from ..utils_executors import get_slurm_file_path
-from ..utils_executors import get_slurm_script_file_path
+from ...filenames import SHUTDOWN_FILENAME
+from ...task_files import TaskFiles
+from ...versions import get_versions
+from ..slurm_common._batching import heuristics
+from ..slurm_common._job_states import STATES_FINISHED
+from ..slurm_common._slurm_config import SlurmConfig
+from ..slurm_common.utils_executors import get_pickle_file_path
+from ..slurm_common.utils_executors import get_slurm_file_path
+from ..slurm_common.utils_executors import get_slurm_script_file_path
 from ._executor_wait_thread import FractalSlurmSSHWaitThread
 from fractal_server.app.runner.components import _COMPONENT_KEY_
 from fractal_server.app.runner.compress_folder import compress_folder
 from fractal_server.app.runner.exceptions import JobExecutionError
 from fractal_server.app.runner.exceptions import TaskExecutionError
-from fractal_server.app.runner.executors.slurm.ssh._slurm_job import SlurmJob
+from fractal_server.app.runner.executors.slurm_ssh._slurm_job import SlurmJob
 from fractal_server.app.runner.extract_archive import extract_archive
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
@@ -533,9 +532,9 @@ class FractalSlurmSSHExecutor(Executor):
                 except AttributeError:
                     actual_component = str(component)
 
-                _task_file_paths = get_task_file_paths(
-                    workflow_dir_local=task_files.workflow_dir_local,
-                    workflow_dir_remote=task_files.workflow_dir_remote,
+                _task_file_paths = TaskFiles(
+                    root_dir_local=task_files.workflow_dir_local,
+                    root_dir_remote=task_files.workflow_dir_remote,
                     task_name=task_files.task_name,
                     task_order=task_files.task_order,
                     component=actual_component,