fractal-server 2.10.4__tar.gz → 2.11.0a0__tar.gz

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.10.4
+Version: 2.11.0a0
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.11.0a0/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.11.0a0"

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/routes/api/v1/dataset.py

@@ -17,7 +17,7 @@ from ....models.v1 import ApplyWorkflow
 from ....models.v1 import Dataset
 from ....models.v1 import Project
 from ....models.v1 import Resource
-from ....runner.filenames import HISTORY_FILENAME
+from ....runner.filenames import HISTORY_FILENAME_V1
 from ....schemas.v1 import DatasetCreateV1
 from ....schemas.v1 import DatasetReadV1
 from ....schemas.v1 import DatasetStatusReadV1
@@ -511,7 +511,7 @@ async def get_workflowtask_status(
         # Highest priority: Read status updates coming from the running-job
         # temporary file. Note: this file only contains information on
         # WorkflowTask's that ran through successfully
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
+        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME_V1
         try:
             with tmp_file.open("r") as f:
                 history = json.load(f)

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/routes/api/v2/status.py

@@ -1,5 +1,3 @@
-import json
-from pathlib import Path
 from typing import Optional
 
 from fastapi import APIRouter
@@ -18,7 +16,6 @@ from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
 
 router = APIRouter()
 
@@ -98,8 +95,8 @@ async def get_workflowtask_status(
     if running_job is None:
         # If no job is running, the chronological-last history item is also the
         # positional-last workflow task to be included in the response.
-        if len(dataset.history) > 0:
-            last_valid_wftask_id = dataset.history[-1]["workflowtask"]["id"]
+        if len(history) > 0:
+            last_valid_wftask_id = history[-1]["workflowtask"]["id"]
         else:
             last_valid_wftask_id = None
     else:
@@ -109,7 +106,24 @@ async def get_workflowtask_status(
         # as "submitted"
         start = running_job.first_task_index
         end = running_job.last_task_index + 1
-        for wftask in workflow.task_list[start:end]:
+
+        running_job_wftasks = workflow.task_list[start:end]
+        running_job_statuses = [
+            workflow_tasks_status_dict.get(wft.id, None)
+            for wft in running_job_wftasks
+        ]
+        try:
+            first_submitted_index = running_job_statuses.index(
+                WorkflowTaskStatusTypeV2.SUBMITTED
+            )
+        except ValueError:
+            logger.warning(
+                f"Job {running_job.id} is submitted but its task list does "
+                f"not contain a {WorkflowTaskStatusTypeV2.SUBMITTED} task."
+            )
+            first_submitted_index = 0
+
+        for wftask in running_job_wftasks[first_submitted_index:]:
             workflow_tasks_status_dict[
                 wftask.id
             ] = WorkflowTaskStatusTypeV2.SUBMITTED
@@ -133,20 +147,6 @@ async def get_workflowtask_status(
             last_valid_wftask_id = None
             logger.warning(f"Now setting {last_valid_wftask_id=}.")
 
-        # Highest priority: Read status updates coming from the running-job
-        # temporary file. Note: this file only contains information on
-        # WorkflowTask's that ran through successfully.
-        tmp_file = Path(running_job.working_dir) / HISTORY_FILENAME
-        try:
-            with tmp_file.open("r") as f:
-                history = json.load(f)
-        except FileNotFoundError:
-            history = []
-        for history_item in history:
-            wftask_id = history_item["workflowtask"]["id"]
-            wftask_status = history_item["status"]
-            workflow_tasks_status_dict[wftask_id] = wftask_status
-
     # Based on previously-gathered information, clean up the response body
     clean_workflow_tasks_status_dict = {}
     for wf_task in workflow.task_list:

fractal_server-2.11.0a0/fractal_server/app/runner/filenames.py

@@ -0,0 +1,4 @@
+HISTORY_FILENAME_V1 = "history.json"
+METADATA_FILENAME_V1 = "metadata.json"
+SHUTDOWN_FILENAME = "shutdown"
+WORKFLOW_LOG_FILENAME = "workflow.log"

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v1/_common.py

@@ -28,8 +28,8 @@ from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-from fractal_server.app.runner.filenames import HISTORY_FILENAME
-from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.filenames import HISTORY_FILENAME_V1
+from fractal_server.app.runner.filenames import METADATA_FILENAME_V1
 from fractal_server.app.runner.task_files import get_task_file_paths
 from fractal_server.string_tools import validate_cmd
 
@@ -610,11 +610,11 @@ def execute_tasks(
         )
 
         # Write most recent metadata to METADATA_FILENAME
-        with open(workflow_dir_local / METADATA_FILENAME, "w") as f:
+        with open(workflow_dir_local / METADATA_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
         # Write most recent metadata to HISTORY_FILENAME
-        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
+        with open(workflow_dir_local / HISTORY_FILENAME_V1, "w") as f:
             json.dump(current_task_pars.history, f, indent=2)
 
     return current_task_pars

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v1/handle_failed_job.py

@@ -24,8 +24,8 @@ from ...models.v1 import Dataset
 from ...models.v1 import Workflow
 from ...models.v1 import WorkflowTask
 from ...schemas.v1 import WorkflowTaskStatusTypeV1
-from ..filenames import HISTORY_FILENAME
-from ..filenames import METADATA_FILENAME
+from ..filenames import HISTORY_FILENAME_V1
+from ..filenames import METADATA_FILENAME_V1
 
 
 def assemble_history_failed_job(
@@ -64,7 +64,7 @@ def assemble_history_failed_job(
     new_history = output_dataset.history
 
     # Part 2: Extend history based on tmp_metadata_file
-    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME
+    tmp_history_file = Path(job.working_dir) / HISTORY_FILENAME_V1
     try:
         with tmp_history_file.open("r") as f:
             tmp_file_history = json.load(f)
@@ -129,7 +129,7 @@ def assemble_meta_failed_job(
     """
 
     new_meta = deepcopy(output_dataset.meta)
-    metadata_file = Path(job.working_dir) / METADATA_FILENAME
+    metadata_file = Path(job.working_dir) / METADATA_FILENAME_V1
     try:
         with metadata_file.open("r") as f:
             metadata_update = json.load(f)

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v2/__init__.py

@@ -11,7 +11,6 @@ from pathlib import Path
 from typing import Optional
 
 from sqlalchemy.orm import Session as DBSyncSession
-from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
 from ....logger import get_logger
@@ -24,7 +23,6 @@ from ....zip_tools import _zip_folder_to_file_and_remove
 from ...db import DB
 from ...models.v2 import DatasetV2
 from ...models.v2 import JobV2
-from ...models.v2 import WorkflowTaskV2
 from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
@@ -38,12 +36,11 @@ from ._local_experimental import (
 )
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
-from .handle_failed_job import assemble_filters_failed_job
-from .handle_failed_job import assemble_history_failed_job
-from .handle_failed_job import assemble_images_failed_job
+from .handle_failed_job import mark_last_wftask_as_failed
 from fractal_server import __VERSION__
 from fractal_server.app.models import UserSettings
 
+
 _backends = {}
 _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_sudo_process_workflow
@@ -115,7 +112,6 @@ async def submit_workflow(
     logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
-
         try:
             job: Optional[JobV2] = db_sync.get(JobV2, job_id)
             dataset: Optional[DatasetV2] = db_sync.get(DatasetV2, dataset_id)
@@ -322,7 +318,7 @@ async def submit_workflow(
         db_sync = next(DB.get_sync_db())
         db_sync.close()
 
-        new_dataset_attributes = await process_workflow(
+        await process_workflow(
             workflow=workflow,
             dataset=dataset,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
@@ -340,14 +336,6 @@ async def submit_workflow(
         )
         logger.debug(f'END workflow "{workflow.name}"')
 
-        # Update dataset attributes, in case of successful execution
-        dataset.history.extend(new_dataset_attributes["history"])
-        dataset.filters = new_dataset_attributes["filters"]
-        dataset.images = new_dataset_attributes["images"]
-        for attribute_name in ["filters", "history", "images"]:
-            flag_modified(dataset, attribute_name)
-        db_sync.merge(dataset)
-
         # Update job DB entry
         job.status = JobStatusTypeV2.DONE
         job.end_timestamp = get_timestamp()
@@ -358,28 +346,13 @@ async def submit_workflow(
         db_sync.commit()
 
     except TaskExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
 
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        failed_wftask = db_sync.get(WorkflowTaskV2, e.workflow_task_id)
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
-            failed_wftask=failed_wftask,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         exception_args_string = "\n".join(e.args)
         log_msg = (
             f"TASK ERROR: "
@@ -390,26 +363,12 @@ async def submit_workflow(
         fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
-
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
-
         fail_job(
             db=db_sync,
             job=job,
@@ -421,27 +380,13 @@ async def submit_workflow(
         )
 
     except Exception:
-
         logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
         logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
-
-        current_traceback = traceback.format_exc()
-
-        # Read dataset attributes produced by the last successful task, and
-        # update the DB dataset accordingly
-        dataset.history = assemble_history_failed_job(
-            job,
-            dataset,
-            workflow,
+        mark_last_wftask_as_failed(
+            dataset_id=dataset_id,
             logger_name=logger_name,
         )
-        latest_filters = assemble_filters_failed_job(job)
-        if latest_filters is not None:
-            dataset.filters = latest_filters
-        latest_images = assemble_images_failed_job(job)
-        if latest_images is not None:
-            dataset.images = latest_images
-        db_sync.merge(dataset)
+        current_traceback = traceback.format_exc()
         fail_job(
             db=db_sync,
             job=job,

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v2/_local/__init__.py

@@ -39,18 +39,15 @@ def _process_workflow(
     workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
-) -> dict:
+) -> None:
     """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalThreadPoolExecutor`.
+    Run the workflow using a `FractalThreadPoolExecutor`.
     """
-
     with FractalThreadPoolExecutor() as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
@@ -58,7 +55,6 @@ def _process_workflow(
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -75,7 +71,7 @@ async def process_workflow(
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Run a workflow
 
@@ -127,11 +123,6 @@ async def process_workflow(
                             (positive exit codes).
         JobExecutionError: wrapper for errors raised by the tasks' executors
                            (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -148,7 +139,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -156,4 +147,3 @@ async def process_workflow(
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-    return new_dataset_attributes

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -21,23 +21,17 @@ def _process_workflow(
     workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
-) -> dict:
+) -> None:
     """
-    Internal processing routine
-
-    Schedules the workflow using a `FractalProcessPoolExecutor`.
-
-    Cf.
-    [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
-    for the call signature.
+    Run the workflow using a `FractalProcessPoolExecutor`.
     """
     with FractalProcessPoolExecutor(
         shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
     ) as executor:
         try:
-            new_dataset_attributes = execute_tasks_v2(
+            execute_tasks_v2(
                 wf_task_list=workflow.task_list[
-                    first_task_index : (last_task_index + 1)  # noqa
+                    first_task_index : (last_task_index + 1)
                 ],
                 dataset=dataset,
                 executor=executor,
@@ -54,8 +48,6 @@ def _process_workflow(
                 )
             )
 
-    return new_dataset_attributes
-
 
 async def process_workflow(
     *,
@@ -71,7 +63,7 @@ async def process_workflow(
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Run a workflow
 
@@ -123,11 +115,6 @@ async def process_workflow(
                             (positive exit codes).
         JobExecutionError: wrapper for errors raised by the tasks' executors
                            (negative exit codes).
-
-    Returns:
-        output_dataset_metadata:
-            The updated metadata for the dataset, as returned by the last task
-            of the workflow
     """
 
     if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
@@ -144,7 +131,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -152,4 +139,3 @@ async def process_workflow(
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-    return new_dataset_attributes

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v2/_slurm_ssh/__init__.py

@@ -17,7 +17,6 @@ This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.
 """
 from pathlib import Path
-from typing import Any
 from typing import Optional
 from typing import Union
 
@@ -47,16 +46,13 @@ def _process_workflow(
     last_task_index: int,
     fractal_ssh: FractalSSH,
     worker_init: Optional[Union[str, list[str]]] = None,
-) -> dict[str, Any]:
+) -> None:
     """
-    Internal processing routine for the SLURM backend
+    Run the workflow using a `FractalSlurmSSHExecutor`.
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
     workflow tasks and returns the new dataset attributes
-
-    Returns:
-        new_dataset_attributes:
     """
 
     if isinstance(worker_init, str):
@@ -80,10 +76,10 @@ def _process_workflow(
         workflow_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
@@ -91,7 +87,6 @@ def _process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -109,7 +104,7 @@ async def process_workflow(
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Process workflow (SLURM backend public interface)
     """
@@ -122,7 +117,7 @@ async def process_workflow(
         last_task_index=last_task_index,
     )
 
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -133,4 +128,3 @@ async def process_workflow(
         worker_init=worker_init,
         fractal_ssh=fractal_ssh,
     )
-    return new_dataset_attributes

{fractal_server-2.10.4 → fractal_server-2.11.0a0}/fractal_server/app/runner/v2/_slurm_sudo/__init__.py

@@ -17,7 +17,6 @@ This backend runs fractal workflows in a SLURM cluster using Clusterfutures
 Executor objects.
 """
 from pathlib import Path
-from typing import Any
 from typing import Optional
 from typing import Union
 
@@ -43,16 +42,13 @@ def _process_workflow(
     slurm_account: Optional[str] = None,
     user_cache_dir: str,
     worker_init: Optional[Union[str, list[str]]] = None,
-) -> dict[str, Any]:
+) -> None:
     """
-    Internal processing routine for the SLURM backend
+    Run the workflow using a `FractalSlurmExecutor`.
 
     This function initialises the a FractalSlurmExecutor, setting logging,
     workflow working dir and user to impersonate. It then schedules the
     workflow tasks and returns the new dataset attributes
-
-    Returns:
-        new_dataset_attributes:
     """
 
     if not slurm_user:
@@ -73,10 +69,10 @@ def _process_workflow(
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
-        new_dataset_attributes = execute_tasks_v2(
+        execute_tasks_v2(
             wf_task_list=workflow.task_list[
-                first_task_index : (last_task_index + 1)  # noqa
-            ],  # noqa
+                first_task_index : (last_task_index + 1)
+            ],
             dataset=dataset,
             executor=executor,
             workflow_dir_local=workflow_dir_local,
@@ -84,7 +80,6 @@ def _process_workflow(
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
         )
-    return new_dataset_attributes
 
 
 async def process_workflow(
@@ -101,7 +96,7 @@ async def process_workflow(
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     worker_init: Optional[str] = None,
-) -> dict:
+) -> None:
     """
     Process workflow (SLURM backend public interface).
     """
@@ -113,8 +108,7 @@ async def process_workflow(
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )
-
-    new_dataset_attributes = await async_wrap(_process_workflow)(
+    await async_wrap(_process_workflow)(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
@@ -127,4 +121,3 @@ async def process_workflow(
         slurm_account=slurm_account,
         worker_init=worker_init,
     )
-    return new_dataset_attributes

fractal_server-2.11.0a0/fractal_server/app/runner/v2/handle_failed_job.py

@@ -0,0 +1,59 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Helper functions to handle Dataset history.
+"""
+import logging
+
+from sqlalchemy.orm.attributes import flag_modified
+
+from ...models.v2 import DatasetV2
+from ...schemas.v2 import WorkflowTaskStatusTypeV2
+from fractal_server.app.db import get_sync_db
+
+
+def mark_last_wftask_as_failed(
+    dataset_id: int,
+    logger_name: str,
+) -> None:
+    """
+    Edit dataset history, by marking last item as failed.
+
+    Args:
+        dataset: The `DatasetV2` object
+        logger_name: A logger name.
+    """
+
+    logger = logging.getLogger(logger_name)
+    with next(get_sync_db()) as db:
+        db_dataset = db.get(DatasetV2, dataset_id)
+        if len(db_dataset.history) == 0:
+            logger.warning(
+                f"History for {dataset_id=} is empty. Likely reason: the job "
+                "failed before its first task was marked as SUBMITTED. "
+                "Continue."
+            )
+            return
+        workflowtask_id = db_dataset.history[-1]["workflowtask"]["id"]
+        last_item_status = db_dataset.history[-1]["status"]
+        if last_item_status != WorkflowTaskStatusTypeV2.SUBMITTED:
+            logger.warning(
+                "Unexpected branch: "
+                f"Last history item, for {workflowtask_id=}, "
+                f"has status {last_item_status}. Skip."
+            )
+            return
+        logger.info(f"Setting history item for {workflowtask_id=} to failed.")
+        db_dataset.history[-1]["status"] = WorkflowTaskStatusTypeV2.FAILED
+        flag_modified(db_dataset, "history")
+        db.merge(db_dataset)
+        db.commit()