fractal-server 2.2.0a1__py3-none-any.whl → 2.3.0a0__py3-none-any.whl

fractal_server/app/runner/executors/slurm/sudo/__init__.py

@@ -0,0 +1,3 @@
+from .executor import SlurmExecutor
+
+__all__ = ["SlurmExecutor"]

fractal_server/app/runner/executors/slurm/{_check_jobs_status.py → sudo/_check_jobs_status.py}

@@ -2,7 +2,7 @@ from subprocess import run  # nosec
 
 from cfut.slurm import STATES_FINISHED
 
-from .....logger import set_logger
+from ......logger import set_logger
 
 
 logger = set_logger(__name__)

fractal_server/app/runner/executors/slurm/{_executor_wait_thread.py → sudo/_executor_wait_thread.py}

@@ -7,7 +7,7 @@ from typing import Optional
 
 from cfut import FileWaitThread
 
-from .....logger import set_logger
+from ......logger import set_logger
 from ._check_jobs_status import _jobs_finished
 
 logger = set_logger(__name__)

fractal_server/app/runner/executors/slurm/{_subprocess_run_as_user.py → sudo/_subprocess_run_as_user.py}

@@ -19,7 +19,7 @@ import shlex
 import subprocess  # nosec
 from typing import Optional
 
-from .....logger import set_logger
+from ......logger import set_logger
 
 logger = set_logger(__name__)
 

fractal_server/app/runner/executors/slurm/{executor.py → sudo/executor.py}

@@ -29,18 +29,18 @@ import cloudpickle
 from cfut import SlurmExecutor
 from cfut.util import random_string
 
-from .....config import get_settings
-from .....logger import set_logger
-from .....syringe import Inject
-from ...exceptions import JobExecutionError
-from ...exceptions import TaskExecutionError
-from ...filenames import SHUTDOWN_FILENAME
-from ...task_files import get_task_file_paths
-from ...task_files import TaskFiles
-from ._batching import heuristics
+from ......config import get_settings
+from ......logger import set_logger
+from ......syringe import Inject
+from ....exceptions import JobExecutionError
+from ....exceptions import TaskExecutionError
+from ....filenames import SHUTDOWN_FILENAME
+from ....task_files import get_task_file_paths
+from ....task_files import TaskFiles
+from ...slurm._slurm_config import get_default_slurm_config
+from ...slurm._slurm_config import SlurmConfig
+from .._batching import heuristics
 from ._executor_wait_thread import FractalSlurmWaitThread
-from ._slurm_config import get_default_slurm_config
-from ._slurm_config import SlurmConfig
 from ._subprocess_run_as_user import _glob_as_user
 from ._subprocess_run_as_user import _glob_as_user_strict
 from ._subprocess_run_as_user import _path_exists_as_user
@@ -1180,7 +1180,7 @@ class FractalSlurmExecutor(SlurmExecutor):
 
         # Prepare SLURM preamble based on SlurmConfig object
         script_lines = slurm_config.to_sbatch_preamble(
-            user_cache_dir=self.user_cache_dir
+            remote_export_dir=self.user_cache_dir
         )
 
         # Extend SLURM preamble with variable which are not in SlurmConfig, and

fractal_server/app/runner/extract_archive.py

@@ -0,0 +1,38 @@
+import sys
+import tarfile
+from pathlib import Path
+
+
+def _remove_suffix(*, string: str, suffix: str) -> str:
+    if string.endswith(suffix):
+        return string[: -len(suffix)]
+    else:
+        raise ValueError(f"Cannot remove {suffix=} from {string=}.")
+
+
+if __name__ == "__main__":
+    help_msg = (
+        "Expected use:\n"
+        "python -m fractal_server.app.runner.extract_archive "
+        "path/to/archive.tar.gz"
+    )
+
+    if len(sys.argv[1:]) != 1:
+        raise ValueError(
+            f"Invalid argument.\n{help_msg}\nProvided: {sys.argv=}"
+        )
+    elif not sys.argv[1].endswith(".tar.gz"):
+        raise ValueError(
+            f"Invalid argument.\n{help_msg}\nProvided: {sys.argv=}"
+        )
+
+    tarfile_path = Path(sys.argv[1])
+
+    print(f"[extract_archive.py] {tarfile_path=}")
+
+    job_folder = tarfile_path.parent
+    subfolder_name = _remove_suffix(string=tarfile_path.name, suffix=".tar.gz")
+    with tarfile.open(tarfile_path) as tar:
+        tar.extractall(path=Path(job_folder, subfolder_name).as_posix())
+
+    print(f"[extract_archive.py] {tarfile_path=}")

fractal_server/app/runner/v1/__init__.py

@@ -22,6 +22,10 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
+from sqlalchemy.orm import Session as DBSyncSession
+
+from ....logger import get_logger
+from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
 from ....utils import get_timestamp
@@ -33,7 +37,7 @@ from ...models.v1 import WorkflowTask
 from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from ..executors.slurm._subprocess_run_as_user import (
+from ..executors.slurm.sudo._subprocess_run_as_user import (
     _mkdir_as_user,
 )
 from ..filenames import WORKFLOW_LOG_FILENAME
@@ -53,6 +57,27 @@ _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_process_workflow
 
 
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: ApplyWorkflow,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV1.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
+
+
 async def submit_workflow(
     *,
     workflow_id: int,
@@ -91,21 +116,41 @@ async def submit_workflow(
             slurm backend.
     """
 
-    # Declare runner backend and set `process_workflow` function
-    settings = Inject(get_settings)
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"ApplyWorkflow {job_id} does not exist")
+            return
+
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_process_workflow
+        else:
+
+            if FRACTAL_RUNNER_BACKEND == "local_experimental":
+                log_msg = (
+                    f"{FRACTAL_RUNNER_BACKEND=} is not available for v1 jobs."
+                )
+            else:
+                log_msg = f"Invalid {FRACTAL_RUNNER_BACKEND=}"
+
+            fail_job(
+                job=job,
+                db=db_sync,
+                log_msg=log_msg,
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        # Declare runner backend and set `process_workflow` function
 
         input_dataset: Dataset = db_sync.get(Dataset, input_dataset_id)
         output_dataset: Dataset = db_sync.get(Dataset, output_dataset_id)
@@ -126,12 +171,9 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV1.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Prepare some of process_workflow arguments
@@ -147,9 +189,14 @@ async def submit_workflow(
         )
 
         if WORKFLOW_DIR_LOCAL.exists():
-            raise RuntimeError(
-                f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
             )
+            return
 
         # Create WORKFLOW_DIR
         original_umask = os.umask(0)
@@ -202,7 +249,6 @@ async def submit_workflow(
         db_sync.refresh(workflow)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
         log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
@@ -302,19 +348,14 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -334,14 +375,13 @@ async def submit_workflow(
         )
 
         db_sync.merge(output_dataset)
-
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
         error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}",
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -364,14 +404,12 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
+        log_msg = (
             f"UNKNOWN ERROR in Fractal job {job.id}\n"
             f"TRACEBACK:\n{current_traceback}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
+
     finally:
         db_sync.close()
+        reset_logger_handlers(logger)

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -22,7 +22,7 @@ from typing import Optional
 from typing import Union
 
 from ...async_wrap import async_wrap
-from ...executors.slurm.executor import FractalSlurmExecutor
+from ...executors.slurm.sudo.executor import FractalSlurmExecutor
 from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
 from ..common import TaskParameters

fractal_server/app/runner/v2/__init__.py

@@ -5,14 +5,18 @@ This module is the single entry point to the runner backend subsystem V2.
 Other subystems should only import this module and not its submodules or
 the individual backends.
 """
+import logging
 import os
 import traceback
 from pathlib import Path
 from typing import Optional
 
+from fabric import Connection  # FIXME SSH: try/except import
+from sqlalchemy.orm import Session as DBSyncSession
 from sqlalchemy.orm.attributes import flag_modified
 
 from ....config import get_settings
+from ....logger import get_logger
 from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
@@ -25,14 +29,15 @@ from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from ..executors.slurm._subprocess_run_as_user import _mkdir_as_user
+from ..executors.slurm.sudo._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
 from ._local_experimental import (
     process_workflow as local_experimental_process_workflow,
 )
-from ._slurm import process_workflow as slurm_process_workflow
+from ._slurm import process_workflow as slurm_sudo_process_workflow
+from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
 from .handle_failed_job import assemble_images_failed_job
@@ -40,8 +45,30 @@ from fractal_server import __VERSION__
 
 _backends = {}
 _backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_sudo_process_workflow
+_backends["slurm_ssh"] = slurm_ssh_process_workflow
 _backends["local_experimental"] = local_experimental_process_workflow
-_backends["slurm"] = slurm_process_workflow
+
+
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: JobV2,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV2.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
 
 
 async def submit_workflow(
@@ -52,6 +79,7 @@ async def submit_workflow(
     worker_init: Optional[str] = None,
     slurm_user: Optional[str] = None,
     user_cache_dir: Optional[str] = None,
+    connection: Optional[Connection] = None,
 ) -> None:
     """
     Prepares a workflow and applies it to a dataset
@@ -78,24 +106,36 @@ async def submit_workflow(
             The username to impersonate for the workflow execution, for the
             slurm backend.
     """
-
     # Declare runner backend and set `process_workflow` function
     settings = Inject(get_settings)
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "local_experimental":
-        process_workflow = local_experimental_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: JobV2 = db_sync.get(JobV2, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"JobV2 {job_id} does not exist")
+            return
+
+        # Declare runner backend and set `process_workflow` function
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        try:
+            process_workflow = _backends[settings.FRACTAL_RUNNER_BACKEND]
+        except KeyError as e:
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"Invalid {FRACTAL_RUNNER_BACKEND=}.\n"
+                    f"Original KeyError: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
 
         dataset: DatasetV2 = db_sync.get(DatasetV2, dataset_id)
         workflow: WorkflowV2 = db_sync.get(WorkflowV2, workflow_id)
@@ -107,31 +147,28 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Define and create server-side working folder
         WORKFLOW_DIR_LOCAL = Path(job.working_dir)
         if WORKFLOW_DIR_LOCAL.exists():
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
+            )
             return
 
         try:
 
-            # Create WORKFLOW_DIR
+            # Create WORKFLOW_DIR_LOCAL
             original_umask = os.umask(0)
             WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
-
             os.umask(original_umask)
 
             # Define and create WORKFLOW_DIR_REMOTE
@@ -146,6 +183,24 @@ async def submit_workflow(
                 _mkdir_as_user(
                     folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user
                 )
+            elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR)
+                    / WORKFLOW_DIR_LOCAL.name
+                )
+                # FIXME SSH: move mkdir to executor, likely within handshake
+
+                from ....ssh._fabric import _mkdir_over_ssh
+
+                _mkdir_over_ssh(
+                    folder=str(WORKFLOW_DIR_REMOTE), connection=connection
+                )
+                logging.info(f"Created {str(WORKFLOW_DIR_REMOTE)} via SSH.")
+            else:
+                logging.error(
+                    "Invalid FRACTAL_RUNNER_BACKEND="
+                    f"{settings.FRACTAL_RUNNER_BACKEND}."
+                )
 
             # Create all tasks subfolders
             for order in range(job.first_task_index, job.last_task_index + 1):
@@ -166,16 +221,20 @@ async def submit_workflow(
                         folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
                         user=slurm_user,
                     )
+                else:
+                    logging.info("Skip remote-subfolder creation")
         except Exception as e:
-            job.status = JobStatusTypeV2.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = (
-                "An error occurred while creating job folder and subfolders.\n"
-                f"Original error: {str(e)}"
+            error_type = type(e).__name__
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=(
+                    f"{error_type} error occurred while creating job folder "
+                    f"and subfolders.\nOriginal error: {str(e)}"
+                ),
+                logger_name=logger_name,
+                emit_log=True,
             )
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
             return
 
         # After Session.commit() is called, either explicitly or when using a
@@ -195,7 +254,6 @@ async def submit_workflow(
             db_sync.refresh(wftask)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
         log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
@@ -207,9 +265,17 @@ async def submit_workflow(
         )
         logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
         logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
-        logger.debug(f"slurm_user: {slurm_user}")
-        logger.debug(f"slurm_account: {job.slurm_account}")
-        logger.debug(f"worker_init: {worker_init}")
+        if FRACTAL_RUNNER_BACKEND == "slurm":
+            logger.debug(f"slurm_user: {slurm_user}")
+            logger.debug(f"slurm_account: {job.slurm_account}")
+            logger.debug(f"worker_init: {worker_init}")
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            logger.debug(f"ssh_host: {settings.FRACTAL_SLURM_SSH_HOST}")
+            logger.debug(f"ssh_user: {settings.FRACTAL_SLURM_SSH_USER}")
+            logger.debug(
+                f"base dir: {settings.FRACTAL_SLURM_SSH_WORKING_BASE_DIR}"
+            )
+            logger.debug(f"worker_init: {worker_init}")
         logger.debug(f"job.id: {job.id}")
         logger.debug(f"job.working_dir: {job.working_dir}")
         logger.debug(f"job.working_dir_user: {job.working_dir_user}")
@@ -218,6 +284,27 @@ async def submit_workflow(
         logger.debug(f'START workflow "{workflow.name}"')
 
     try:
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+            backend_specific_kwargs = {}
+        elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+            process_workflow = local_experimental_process_workflow
+            backend_specific_kwargs = {}
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_sudo_process_workflow
+            backend_specific_kwargs = dict(
+                slurm_user=slurm_user,
+                slurm_account=job.slurm_account,
+                user_cache_dir=user_cache_dir,
+            )
+        elif FRACTAL_RUNNER_BACKEND == "slurm_ssh":
+            process_workflow = slurm_ssh_process_workflow
+            backend_specific_kwargs = dict(connection=connection)
+        else:
+            raise RuntimeError(
+                f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}"
+            )
+
         # "The Session.close() method does not prevent the Session from being
         # used again. The Session itself does not actually have a distinct
         # “closed” state; it merely means the Session will release all database
@@ -234,15 +321,13 @@ async def submit_workflow(
         new_dataset_attributes = await process_workflow(
             workflow=workflow,
             dataset=dataset,
-            slurm_user=slurm_user,
-            slurm_account=job.slurm_account,
-            user_cache_dir=user_cache_dir,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
             workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,
             last_task_index=job.last_task_index,
+            **backend_specific_kwargs,
         )
 
         logger.info(
@@ -291,18 +376,14 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -325,12 +406,15 @@ async def submit_workflow(
             dataset.images = latest_images
         db_sync.merge(dataset)
 
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"JOB ERROR in Fractal job {job.id}:\n"
+                f"TRACEBACK:\n{e.assemble_error()}"
+            ),
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -354,15 +438,16 @@ async def submit_workflow(
         if latest_images is not None:
             dataset.images = latest_images
         db_sync.merge(dataset)
-
-        job.status = JobStatusTypeV2.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
-            f"UNKNOWN ERROR in Fractal job {job.id}\n"
-            f"TRACEBACK:\n{current_traceback}"
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=(
+                f"UNKNOWN ERROR in Fractal job {job.id}\n"
+                f"TRACEBACK:\n{current_traceback}"
+            ),
+            logger_name=logger_name,
         )
-        db_sync.merge(job)
-        db_sync.commit()
+
     finally:
         reset_logger_handlers(logger)
         db_sync.close()