fractal-server 2.10.5__py3-none-any.whl → 2.11.0a2__py3-none-any.whl

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -11,6 +11,7 @@ from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _local_submit_setup
 from .executor import FractalProcessPoolExecutor
+from fractal_server.images.models import AttributeFiltersType
 
 
 def _process_workflow(
@@ -21,23 +22,18 @@ def _process_workflow(
     workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
-) -> dict:
+    job_attribute_filters: AttributeFiltersType,
+) -> None:
     """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalProcessPoolExecutor`.
-
-    Cf.
-    [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
-    for the call signature.
+    Run the workflow using a `FractalProcessPoolExecutor`.
     """
     with FractalProcessPoolExecutor(
         shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
     ) as executor:
         try:
-            new_dataset_attributes = execute_tasks_v2(
+            execute_tasks_v2(
                 wf_task_list=workflow.task_list[
-                    first_task_index : (last_task_index + 1)  # noqa
+                    first_task_index : (last_task_index + 1)
                 ],
                 dataset=dataset,
                 executor=executor,
@@ -45,6 +41,7 @@ def _process_workflow(
                 workflow_dir_remote=workflow_dir_local,
                 logger_name=logger_name,
                 submit_setup_call=_local_submit_setup,
+                job_attribute_filters=job_attribute_filters,
             )
         except BrokenProcessPool as e:
             raise JobExecutionError(
@@ -54,8 +51,6 @@ def _process_workflow(
                 )
             )
 
-    return new_dataset_attributes
-
 
 async def process_workflow(
     *,
@@ -66,12 +61,13 @@ async def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Run a workflow
 
@@ -123,11 +119,6 @@ async def process_workflow(
                             (positive exit codes).
         JobExecutionError: wrapper for errors raised by the tasks' executors
                            (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -144,12 +135,12 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
         workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
+        job_attribute_filters=job_attribute_filters,
     )
-    return new_dataset_attributes

fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -17,7 +17,6 @@ This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.
 """
 from pathlib import Path
-from typing import Any
 from typing import Optional
 from typing import Union
 
@@ -30,9 +29,9 @@ from ...executors.slurm.ssh.executor import FractalSlurmSSHExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
+from fractal_server.images.models import AttributeFiltersType
 from fractal_server.logger import set_logger
 
-
 logger = set_logger(__name__)
 
 
@@ -47,16 +46,14 @@ def _process_workflow(
     last_task_index: int,
     fractal_ssh: FractalSSH,
     worker_init: Optional[Union[str, list[str]]] = None,
-) -> dict[str, Any]:
+    job_attribute_filters: AttributeFiltersType,
+) -> None:
     """
-    Internal processing routine for the SLURM backend
+    Run the workflow using a `FractalSlurmSSHExecutor`.
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
     workflow tasks and returns the new dataset attributes
-
-    Returns:
-        new_dataset_attributes:
     """
 
     if isinstance(worker_init, str):
@@ -80,18 +77,18 @@ def _process_workflow(
         workflow_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
+            job_attribute_filters=job_attribute_filters,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -103,13 +100,14 @@ async def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
-    # Not used
+    job_attribute_filters: AttributeFiltersType,
     fractal_ssh: FractalSSH,
+    worker_init: Optional[str] = None,
+    # Not used
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
-    worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Process workflow (SLURM backend public interface)
     """
@@ -122,7 +120,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -132,5 +130,5 @@ async def process_workflow(
         last_task_index=last_task_index,
         worker_init=worker_init,
         fractal_ssh=fractal_ssh,
+        job_attribute_filters=job_attribute_filters,
     )
-    return new_dataset_attributes

fractal_server/app/runner/v2/_slurm_sudo/__init__.py

@@ -17,7 +17,6 @@ This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.
 """
 from pathlib import Path
-from typing import Any
 from typing import Optional
 from typing import Union
 
@@ -28,6 +27,7 @@ from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from ..runner import execute_tasks_v2
 from ._submit_setup import _slurm_submit_setup
+from fractal_server.images.models import AttributeFiltersType
 
 
 def _process_workflow(
@@ -43,16 +43,14 @@ def _process_workflow(
     slurm_account: Optional[str] = None,
     user_cache_dir: str,
     worker_init: Optional[Union[str, list[str]]] = None,
-) -> dict[str, Any]:
+    job_attribute_filters: AttributeFiltersType,
+) -> None:
     """
-    Internal processing routine for the SLURM backend
+    Run the workflow using a `FractalSlurmExecutor`.
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
     workflow tasks and returns the new dataset attributes
-
-    Returns:
-        new_dataset_attributes:
     """
 
     if not slurm_user:
@@ -73,18 +71,18 @@ def _process_workflow(
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
+            job_attribute_filters=job_attribute_filters,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -96,12 +94,13 @@ async def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Process workflow (SLURM backend public interface).
     """
@@ -113,8 +112,7 @@ async def process_workflow(
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -126,5 +124,5 @@ async def process_workflow(
         slurm_user=slurm_user,
         slurm_account=slurm_account,
         worker_init=worker_init,
+        job_attribute_filters=job_attribute_filters,
     )
-    return new_dataset_attributes

fractal_server/app/runner/v2/handle_failed_job.py

@@ -12,147 +12,48 @@
 """
 Helper functions to handle Dataset history.
 """
-import json
 import logging
-from pathlib import Path
-from typing import Any
-from typing import Optional
+
+from sqlalchemy.orm.attributes import flag_modified
 
 from ...models.v2 import DatasetV2
-from ...models.v2 import JobV2
-from ...models.v2 import WorkflowTaskV2
-from ...models.v2 import WorkflowV2
 from ...schemas.v2 import WorkflowTaskStatusTypeV2
-from ..filenames import FILTERS_FILENAME
-from ..filenames import HISTORY_FILENAME
-from ..filenames import IMAGES_FILENAME
+from fractal_server.app.db import get_sync_db
 
 
-def assemble_history_failed_job(
-    job: JobV2,
-    dataset: DatasetV2,
-    workflow: WorkflowV2,
-    logger_name: Optional[str] = None,
-    failed_wftask: Optional[WorkflowTaskV2] = None,
-) -> list[dict[str, Any]]:
+def mark_last_wftask_as_failed(
+    dataset_id: int,
+    logger_name: str,
+) -> None:
     """
-    Assemble `history` after a workflow-execution job fails.
+    Edit dataset history, by marking last item as failed.
 
     Args:
-        job:
-            The failed `JobV2` object.
-        dataset:
-            The `DatasetV2` object associated to `job`.
-        workflow:
-            The `WorkflowV2` object associated to `job`.
+        dataset: The `DatasetV2` object
         logger_name: A logger name.
-        failed_wftask:
-            If set, append it to `history` during step 3; if `None`, infer
-            it by comparing the job task list and the one in
-            `HISTORY_FILENAME`.
-
-    Returns:
-        The new value of `history`, to be merged into
-        `dataset.meta`.
     """
 
     logger = logging.getLogger(logger_name)
-
-    # The final value of the history attribute should include up to three
-    # parts, coming from: the database, the temporary file, the failed-task
-    # information.
-
-    # Part 1: Read exising history from DB
-    new_history = dataset.history
-
-    # Part 2: Extend history based on temporary-file contents
-    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
-    try:
-        with tmp_history_file.open("r") as f:
-            tmp_file_history = json.load(f)
-            new_history.extend(tmp_file_history)
-    except FileNotFoundError:
-        tmp_file_history = []
-
-    # Part 3/A: Identify failed task, if needed
-    if failed_wftask is None:
-        job_wftasks = workflow.task_list[
-            job.first_task_index : (job.last_task_index + 1)  # noqa
-        ]
-        tmp_file_wftasks = [
-            history_item["workflowtask"] for history_item in tmp_file_history
-        ]
-        if len(job_wftasks) <= len(tmp_file_wftasks):
-            n_tasks_job = len(job_wftasks)
-            n_tasks_tmp = len(tmp_file_wftasks)
-            logger.error(
-                "Cannot identify the failed task based on job task list "
-                f"(length {n_tasks_job}) and temporary-file task list "
-                f"(length {n_tasks_tmp})."
+    with next(get_sync_db()) as db:
+        db_dataset = db.get(DatasetV2, dataset_id)
+        if len(db_dataset.history) == 0:
+            logger.warning(
+                f"History for {dataset_id=} is empty. Likely reason: the job "
+                "failed before its first task was marked as SUBMITTED. "
+                "Continue."
             )
-            logger.error("Failed task not appended to history.")
-        else:
-            failed_wftask = job_wftasks[len(tmp_file_wftasks)]
-
-    # Part 3/B: Append failed task to history
-    if failed_wftask is not None:
-        failed_wftask_dump = failed_wftask.model_dump(exclude={"task"})
-        failed_wftask_dump["task"] = failed_wftask.task.model_dump()
-        new_history_item = dict(
-            workflowtask=failed_wftask_dump,
-            status=WorkflowTaskStatusTypeV2.FAILED,
-            parallelization=dict(),  # FIXME: re-include parallelization
-        )
-        new_history.append(new_history_item)
-
-    return new_history
-
-
-def assemble_images_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
-    """
-    Assemble `DatasetV2.images` for a failed workflow-execution.
-
-    Assemble new value of `images` based on the last successful task, i.e.
-    based on the content of the temporary `IMAGES_FILENAME` file. If the file
-    is missing, return `None`.
-
-    Argumentss:
-        job:
-            The failed `JobV2` object.
-
-    Returns:
-        The new value of `dataset.images`, or `None` if `IMAGES_FILENAME`
-        is missing.
-    """
-    tmp_file = Path(job.working_dir) / IMAGES_FILENAME
-    try:
-        with tmp_file.open("r") as f:
-            new_images = json.load(f)
-        return new_images
-    except FileNotFoundError:
-        return None
-
-
-def assemble_filters_failed_job(job: JobV2) -> Optional[dict[str, Any]]:
-    """
-    Assemble `DatasetV2.filters` for a failed workflow-execution.
-
-    Assemble new value of `filters` based on the last successful task, i.e.
-    based on the content of the temporary `FILTERS_FILENAME` file. If the file
-    is missing, return `None`.
-
-    Argumentss:
-        job:
-            The failed `JobV2` object.
-
-    Returns:
-        The new value of `dataset.filters`, or `None` if `FILTERS_FILENAME`
-        is missing.
-    """
-    tmp_file = Path(job.working_dir) / FILTERS_FILENAME
-    try:
-        with tmp_file.open("r") as f:
-            new_filters = json.load(f)
-        return new_filters
-    except FileNotFoundError:
-        return None
+            return
+        workflowtask_id = db_dataset.history[-1]["workflowtask"]["id"]
+        last_item_status = db_dataset.history[-1]["status"]
+        if last_item_status != WorkflowTaskStatusTypeV2.SUBMITTED:
+            logger.warning(
+                "Unexpected branch: "
+                f"Last history item, for {workflowtask_id=}, "
+                f"has status {last_item_status}. Skip."
+            )
+            return
+        logger.info(f"Setting history item for {workflowtask_id=} to failed.")
+        db_dataset.history[-1]["status"] = WorkflowTaskStatusTypeV2.FAILED
+        flag_modified(db_dataset, "history")
+        db.merge(db_dataset)
+        db.commit()

fractal_server/app/runner/v2/merge_outputs.py

@@ -1,38 +1,35 @@
-from copy import copy
-
 from fractal_server.app.runner.v2.deduplicate_list import deduplicate_list
 from fractal_server.app.runner.v2.task_interface import TaskOutput
 
 
 def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
 
+    if len(task_outputs) == 0:
+        return TaskOutput()
+
     final_image_list_updates = []
     final_image_list_removals = []
-    last_new_filters = None
 
-    for ind, task_output in enumerate(task_outputs):
+    for task_output in task_outputs:
 
         final_image_list_updates.extend(task_output.image_list_updates)
         final_image_list_removals.extend(task_output.image_list_removals)
 
-        # Check that all filters are the same
-        current_new_filters = task_output.filters
-        if ind == 0:
-            last_new_filters = copy(current_new_filters)
-        if current_new_filters != last_new_filters:
-            raise ValueError(f"{current_new_filters=} but {last_new_filters=}")
-        last_new_filters = copy(current_new_filters)
+        # Check that all type_filters are the same
+        if task_output.type_filters != task_outputs[0].type_filters:
+            raise ValueError(
+                f"{task_output.type_filters=} "
+                f"but {task_outputs[0].type_filters=}"
+            )
 
+    # Note: the ordering of `image_list_removals` is not guaranteed
     final_image_list_updates = deduplicate_list(final_image_list_updates)
-
-    additional_args = {}
-    if last_new_filters is not None:
-        additional_args["filters"] = last_new_filters
+    final_image_list_removals = list(set(final_image_list_removals))
 
     final_output = TaskOutput(
         image_list_updates=final_image_list_updates,
         image_list_removals=final_image_list_removals,
-        **additional_args,
+        type_filters=task_outputs[0].type_filters,
     )
 
     return final_output