fractal-server 1.4.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/runner/v1/__init__.py

@@ -0,0 +1,366 @@
+# Copyright 2022 (C) Friedrich Miescher Institute for Biomedical Research and
+# University of Zurich
+#
+# Original authors:
+# Jacopo Nespolo <jacopo.nespolo@exact-lab.it>
+# Tommaso Comparin <tommaso.comparin@exact-lab.it>
+# Marco Franzon <marco.franzon@exact-lab.it>
+#
+# This file is part of Fractal and was originally developed by eXact lab S.r.l.
+# <exact-lab.it> under contract with Liberali Lab from the Friedrich Miescher
+# Institute for Biomedical Research and Pelkmans Lab from the University of
+# Zurich.
+"""
+Runner backend subsystem root
+
+This module is the single entry point to the runner backend subsystem. Other
+subystems should only import this module and not its submodules or the
+individual backends.
+"""
+import os
+import traceback
+from pathlib import Path
+from typing import Optional
+
+from ....logger import set_logger
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import DB
+from ...models.v1 import ApplyWorkflow
+from ...models.v1 import Dataset
+from ...models.v1 import Workflow
+from ...models.v1 import WorkflowTask
+from ...schemas.v1 import JobStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..filenames import WORKFLOW_LOG_FILENAME
+from ._local import process_workflow as local_process_workflow
+from ._slurm import process_workflow as slurm_process_workflow
+from .common import close_job_logger
+from .common import validate_workflow_compatibility  # noqa: F401
+from .handle_failed_job import assemble_history_failed_job
+from .handle_failed_job import assemble_meta_failed_job
+from fractal_server import __VERSION__
+from fractal_server.config import get_settings
+
+
+_backends = {}
+_backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_process_workflow
+
+
+async def submit_workflow(
+    *,
+    workflow_id: int,
+    input_dataset_id: int,
+    output_dataset_id: int,
+    job_id: int,
+    worker_init: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    user_cache_dir: Optional[str] = None,
+) -> None:
+    """
+    Prepares a workflow and applies it to a dataset
+
+    This function wraps the process_workflow one, which is different for each
+    backend (e.g. local or slurm backend).
+
+    Args:
+        workflow_id:
+            ID of the workflow being applied
+        input_dataset_id:
+            Input dataset ID
+        output_dataset_id:
+            ID of the destination dataset of the workflow.
+        job_id:
+            Id of the job record which stores the state for the current
+            workflow application.
+        worker_init:
+            Custom executor parameters that get parsed before the execution of
+            each task.
+        user_cache_dir:
+            Cache directory (namely a path where the user can write); for the
+            slurm backend, this is used as a base directory for
+            `job.working_dir_user`.
+        slurm_user:
+            The username to impersonate for the workflow execution, for the
+            slurm backend.
+    """
+
+    # Declare runner backend and set `process_workflow` function
+    settings = Inject(get_settings)
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    if FRACTAL_RUNNER_BACKEND == "local":
+        process_workflow = local_process_workflow
+    elif FRACTAL_RUNNER_BACKEND == "slurm":
+        process_workflow = slurm_process_workflow
+    else:
+        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+
+    with next(DB.get_sync_db()) as db_sync:
+
+        job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
+        if not job:
+            raise ValueError(f"Cannot fetch job {job_id} from database")
+
+        input_dataset: Dataset = db_sync.get(Dataset, input_dataset_id)
+        output_dataset: Dataset = db_sync.get(Dataset, output_dataset_id)
+        workflow: Workflow = db_sync.get(Workflow, workflow_id)
+        if not (input_dataset and output_dataset and workflow):
+            log_msg = ""
+            if not input_dataset:
+                log_msg += (
+                    f"Cannot fetch input_dataset {input_dataset_id} "
+                    "from database\n"
+                )
+            if not output_dataset:
+                log_msg += (
+                    f"Cannot fetch output_dataset {output_dataset_id} "
+                    "from database\n"
+                )
+            if not workflow:
+                log_msg += (
+                    f"Cannot fetch workflow {workflow_id} from database\n"
+                )
+            job.status = JobStatusTypeV1.FAILED
+            job.end_timestamp = get_timestamp()
+            job.log = log_msg
+            db_sync.merge(job)
+            db_sync.commit()
+            db_sync.close()
+            return
+
+        # Prepare some of process_workflow arguments
+        input_paths = input_dataset.paths
+        output_path = output_dataset.paths[0]
+
+        # Define and create server-side working folder
+        project_id = workflow.project_id
+        timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+        WORKFLOW_DIR = (
+            settings.FRACTAL_RUNNER_WORKING_BASE_DIR
+            / (
+                f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
+                f"_{timestamp_string}"
+            )
+        ).resolve()
+
+        if WORKFLOW_DIR.exists():
+            raise RuntimeError(f"Workflow dir {WORKFLOW_DIR} already exists.")
+
+        # Create WORKFLOW_DIR with 755 permissions
+        original_umask = os.umask(0)
+        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
+        os.umask(original_umask)
+
+        # Define and create user-side working folder, if needed
+        if FRACTAL_RUNNER_BACKEND == "local":
+            WORKFLOW_DIR_USER = WORKFLOW_DIR
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+
+            from ..executors.slurm._subprocess_run_as_user import (
+                _mkdir_as_user,
+            )
+
+            WORKFLOW_DIR_USER = (
+                Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
+            ).resolve()
+            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
+        else:
+            raise ValueError(f"{FRACTAL_RUNNER_BACKEND=} not supported")
+
+        # Update db
+        job.working_dir = WORKFLOW_DIR.as_posix()
+        job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+        db_sync.merge(job)
+        db_sync.commit()
+
+        # After Session.commit() is called, either explicitly or when using a
+        # context manager, all objects associated with the Session are expired.
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_basics.html#opening-and-closing-a-session
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_state_management.html#refreshing-expiring
+
+        # See issue #928:
+        # https://github.com/fractal-analytics-platform/
+        #   fractal-server/issues/928
+
+        db_sync.refresh(input_dataset)
+        db_sync.refresh(output_dataset)
+        db_sync.refresh(workflow)
+
+        # Write logs
+        logger_name = f"WF{workflow_id}_job{job_id}"
+        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        logger = set_logger(
+            logger_name=logger_name,
+            log_file_path=log_file_path,
+        )
+        logger.info(
+            f'Start execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
+        logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
+        logger.debug(f"slurm_user: {slurm_user}")
+        logger.debug(f"slurm_account: {job.slurm_account}")
+        logger.debug(f"worker_init: {worker_init}")
+        logger.debug(f"input metadata keys: {list(input_dataset.meta.keys())}")
+        logger.debug(f"input_paths: {input_paths}")
+        logger.debug(f"output_path: {output_path}")
+        logger.debug(f"job.id: {job.id}")
+        logger.debug(f"job.working_dir: {job.working_dir}")
+        logger.debug(f"job.working_dir_user: {job.working_dir_user}")
+        logger.debug(f"job.first_task_index: {job.first_task_index}")
+        logger.debug(f"job.last_task_index: {job.last_task_index}")
+        logger.debug(f'START workflow "{workflow.name}"')
+
+    try:
+        # "The Session.close() method does not prevent the Session from being
+        # used again. The Session itself does not actually have a distinct
+        # “closed” state; it merely means the Session will release all database
+        # connections and ORM objects."
+        # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close).
+        #
+        # We close the session before the (possibly long) process_workflow
+        # call, to make sure all DB connections are released. The reason why we
+        # are not using a context manager within the try block is that we also
+        # need access to db_sync in the except branches.
+        db_sync = next(DB.get_sync_db())
+        db_sync.close()
+
+        output_dataset_meta_hist = await process_workflow(
+            workflow=workflow,
+            input_paths=input_paths,
+            output_path=output_path,
+            input_metadata=input_dataset.meta,
+            input_history=input_dataset.history,
+            slurm_user=slurm_user,
+            slurm_account=job.slurm_account,
+            user_cache_dir=user_cache_dir,
+            workflow_dir=WORKFLOW_DIR,
+            workflow_dir_user=WORKFLOW_DIR_USER,
+            logger_name=logger_name,
+            worker_init=worker_init,
+            first_task_index=job.first_task_index,
+            last_task_index=job.last_task_index,
+        )
+
+        logger.info(
+            f'End execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f'END workflow "{workflow.name}"')
+
+        # Replace output_dataset.meta and output_dataset.history with their
+        # up-to-date versions, obtained within process_workflow
+        output_dataset.history = output_dataset_meta_hist.pop("history")
+        output_dataset.meta = output_dataset_meta_hist.pop("metadata")
+
+        db_sync.merge(output_dataset)
+
+        # Update job DB entry
+        job.status = JobStatusTypeV1.DONE
+        job.end_timestamp = get_timestamp()
+        with log_file_path.open("r") as f:
+            logs = f.read()
+        job.log = logs
+        db_sync.merge(job)
+        close_job_logger(logger)
+        db_sync.commit()
+
+    except TaskExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
+
+        # Assemble output_dataset.meta based on the last successful task, i.e.
+        # based on METADATA_FILENAME
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
+
+        # Assemble new history and assign it to output_dataset.meta
+        failed_wftask = db_sync.get(WorkflowTask, e.workflow_task_id)
+        output_dataset.history = assemble_history_failed_job(
+            job,
+            output_dataset,
+            workflow,
+            logger,
+            failed_wftask=failed_wftask,
+        )
+
+        db_sync.merge(output_dataset)
+
+        job.status = JobStatusTypeV1.FAILED
+        job.end_timestamp = get_timestamp()
+
+        exception_args_string = "\n".join(e.args)
+        job.log = (
+            f"TASK ERROR: "
+            f"Task name: {e.task_name}, "
+            f"position in Workflow: {e.workflow_task_order}\n"
+            f"TRACEBACK:\n{exception_args_string}"
+        )
+        db_sync.merge(job)
+        close_job_logger(logger)
+        db_sync.commit()
+
+    except JobExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
+
+        # Assemble output_dataset.meta based on the last successful task, i.e.
+        # based on METADATA_FILENAME
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
+
+        # Assemble new history and assign it to output_dataset.meta
+        output_dataset.history = assemble_history_failed_job(
+            job,
+            output_dataset,
+            workflow,
+            logger,
+        )
+
+        db_sync.merge(output_dataset)
+
+        job.status = JobStatusTypeV1.FAILED
+        job.end_timestamp = get_timestamp()
+        error = e.assemble_error()
+        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
+        db_sync.merge(job)
+        close_job_logger(logger)
+        db_sync.commit()
+
+    except Exception:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
+        logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
+
+        current_traceback = traceback.format_exc()
+
+        # Assemble output_dataset.meta based on the last successful task, i.e.
+        # based on METADATA_FILENAME
+        output_dataset.meta = assemble_meta_failed_job(job, output_dataset)
+
+        # Assemble new history and assign it to output_dataset.meta
+        output_dataset.history = assemble_history_failed_job(
+            job,
+            output_dataset,
+            workflow,
+            logger,
+        )
+
+        db_sync.merge(output_dataset)
+
+        job.status = JobStatusTypeV1.FAILED
+        job.end_timestamp = get_timestamp()
+        job.log = (
+            f"UNKNOWN ERROR in Fractal job {job.id}\n"
+            f"TRACEBACK:\n{current_traceback}"
+        )
+        db_sync.merge(job)
+        close_job_logger(logger)
+        db_sync.commit()
+    finally:
+        db_sync.close()

fractal_server/app/runner/{_common.py → v1/_common.py}

@@ -11,7 +11,6 @@ import subprocess  # nosec
 import traceback
 from concurrent.futures import Executor
 from copy import deepcopy
-from functools import lru_cache
 from functools import partial
 from pathlib import Path
 from shlex import split as shlex_split
@@ -19,22 +18,19 @@ from typing import Any
 from typing import Callable
 from typing import Optional
 
-from ...config import get_settings
-from ...logger import get_logger
-from ...syringe import Inject
-from ..models import Task
-from ..models import WorkflowTask
-from ..schemas import WorkflowTaskStatusType
-from .common import JobExecutionError
-from .common import TaskExecutionError
+from ....config import get_settings
+from ....logger import get_logger
+from ....syringe import Inject
+from ...models.v1 import Task
+from ...models.v1 import WorkflowTask
+from ...schemas.v1 import WorkflowTaskStatusTypeV1
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
 from .common import TaskParameters
 from .common import write_args_file
-
-
-HISTORY_FILENAME = "history.json"
-METADATA_FILENAME = "metadata.json"
-SHUTDOWN_FILENAME = "shutdown"
-WORKFLOW_LOG_FILENAME = "workflow.log"
+from fractal_server.app.runner.filenames import HISTORY_FILENAME
+from fractal_server.app.runner.filenames import METADATA_FILENAME
+from fractal_server.app.runner.task_files import get_task_file_paths
 
 
 def no_op_submit_setup_call(
@@ -42,7 +38,6 @@ def no_op_submit_setup_call(
     wftask: WorkflowTask,
     workflow_dir: Path,
     workflow_dir_user: Path,
-    task_pars: TaskParameters,
 ) -> dict:
     """
     Default (no-operation) interface of submit_setup_call.
@@ -50,14 +45,6 @@ def no_op_submit_setup_call(
     return {}
 
 
-def sanitize_component(value: str) -> str:
-    """
-    Remove {" ", "/", "."} form a string, e.g. going from
-    'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
-    """
-    return value.replace(" ", "_").replace("/", "_").replace(".", "_")
-
-
 def _task_needs_image_list(_task: Task) -> bool:
     """
     Whether a task requires `metadata["image"]` in its `args.json` file.
@@ -78,98 +65,6 @@ def _task_needs_image_list(_task: Task) -> bool:
         return False
 
 
-class TaskFiles:
-    """
-    Group all file paths pertaining to a task
-
-    Attributes:
-        workflow_dir:
-            Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
-        task_order:
-            Positional order of the task within a workflow.
-        component:
-            Specific component to run the task for (relevant for tasks that
-            will be executed in parallel over many components).
-        file_prefix:
-            Prefix for all task-related files.
-        args:
-            Path for input json file.
-        metadiff:
-            Path for output json file with metadata update.
-        out:
-            Path for task-execution stdout.
-        err:
-            Path for task-execution stderr.
-    """
-
-    workflow_dir: Path
-    workflow_dir_user: Path
-    task_order: Optional[int] = None
-    component: Optional[str] = None
-
-    file_prefix: str
-    args: Path
-    out: Path
-    err: Path
-    metadiff: Path
-
-    def __init__(
-        self,
-        workflow_dir: Path,
-        workflow_dir_user: Path,
-        task_order: Optional[int] = None,
-        component: Optional[str] = None,
-    ):
-        self.workflow_dir = workflow_dir
-        self.workflow_dir_user = workflow_dir_user
-        self.task_order = task_order
-        self.component = component
-
-        if self.component is not None:
-            component_safe = sanitize_component(str(self.component))
-            component_safe = f"_par_{component_safe}"
-        else:
-            component_safe = ""
-
-        if self.task_order is not None:
-            order = str(self.task_order)
-        else:
-            order = "task"
-        self.file_prefix = f"{order}{component_safe}"
-        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
-        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
-        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
-        self.metadiff = (
-            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
-        )
-
-
-@lru_cache()
-def get_task_file_paths(
-    workflow_dir: Path,
-    workflow_dir_user: Path,
-    task_order: Optional[int] = None,
-    component: Optional[str] = None,
-) -> TaskFiles:
-    """
-    Return the corrisponding TaskFiles object
-
-    This function is mainly used as a cache to avoid instantiating needless
-    objects.
-    """
-    return TaskFiles(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
-        task_order=task_order,
-        component=component,
-    )
-
-
 def _call_command_wrapper(cmd: str, stdout: Path, stderr: Path) -> None:
     """
     Call a command and write its stdout and stderr to files
@@ -312,7 +207,7 @@ def call_single_task(
         with task_files.metadiff.open("r") as f_metadiff:
             diff_metadata = json.load(f_metadiff)
     except FileNotFoundError as e:
-        logger.error(
+        logger.warning(
             f"Skip collection of updated metadata. Original error: {str(e)}"
         )
         diff_metadata = {}
@@ -331,7 +226,7 @@ def call_single_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=None,
     )
     updated_history = task_pars.history.copy()
@@ -529,7 +424,6 @@ def call_parallel_task(
     try:
         extra_setup = submit_setup_call(
             wftask=wftask,
-            task_pars=task_pars_depend,
             workflow_dir=workflow_dir,
             workflow_dir_user=workflow_dir_user,
         )
@@ -592,7 +486,7 @@ def call_parallel_task(
     wftask_dump["task"] = wftask.task.model_dump()
     new_history_item = dict(
         workflowtask=wftask_dump,
-        status=WorkflowTaskStatusType.DONE,
+        status=WorkflowTaskStatusTypeV1.DONE,
         parallelization=dict(
             parallelization_level=wftask.parallelization_level,
             component_list=component_list,
@@ -681,7 +575,6 @@ def execute_tasks(
             try:
                 extra_setup = submit_setup_call(
                     wftask=this_wftask,
-                    task_pars=current_task_pars,
                     workflow_dir=workflow_dir,
                     workflow_dir_user=workflow_dir_user,
                 )

fractal_server/app/runner/{_local → v1/_local}/__init__.py

@@ -23,10 +23,10 @@ from pathlib import Path
 from typing import Any
 from typing import Optional
 
-from ...models import Workflow
+from ....models.v1 import Workflow
+from ...async_wrap import async_wrap
+from ...set_start_and_last_task_index import set_start_and_last_task_index
 from .._common import execute_tasks
-from ..common import async_wrap
-from ..common import set_start_and_last_task_index
 from ..common import TaskParameters
 from ._submit_setup import _local_submit_setup
 from .executor import FractalThreadPoolExecutor
@@ -49,7 +49,8 @@ def _process_workflow(
 
     Schedules the workflow using a `FractalThreadPoolExecutor`.
 
-    Cf. [process_workflow][fractal_server.app.runner._local.process_workflow]
+    Cf.
+    [process_workflow][fractal_server.app.runner.v1._local.process_workflow]
     for the call signature.
     """
 

fractal_server/app/runner/{_local → v1/_local}/_local_config.py

@@ -19,9 +19,9 @@ from pydantic import BaseModel
 from pydantic import Extra
 from pydantic.error_wrappers import ValidationError
 
-from ....config import get_settings
-from ....syringe import Inject
-from ...models import WorkflowTask
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v1 import WorkflowTask
 
 
 class LocalBackendConfigError(ValueError):
@@ -63,15 +63,14 @@ def get_local_backend_config(
     The sources for `parallel_tasks_per_job` attributes, starting from the
     highest-priority one, are
 
-    1. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
-       `Workflow.insert_task`, also includes `wftask.task.meta`);
+    1. Properties in `wftask.meta`;
     2. The general content of the local-backend configuration file;
     3. The default value (`None`).
 
     Arguments:
         wftask:
-            WorkflowTask for which the backend configuration is is to be
-            prepared.
+            WorkflowTask (V1) for which the backend configuration should
+            be prepared.
         config_path:
             Path of local-backend configuration file; if `None`, use
             `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.

fractal_server/app/runner/{_local → v1/_local}/_submit_setup.py

@@ -14,8 +14,7 @@ Submodule to define _local_submit_setup
 from pathlib import Path
 from typing import Optional
 
-from ...models import WorkflowTask
-from ..common import TaskParameters
+from ....models.v1 import WorkflowTask
 from ._local_config import get_local_backend_config
 
 
@@ -24,7 +23,6 @@ def _local_submit_setup(
     wftask: WorkflowTask,
     workflow_dir: Optional[Path] = None,
     workflow_dir_user: Optional[Path] = None,
-    task_pars: Optional[TaskParameters] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -33,8 +31,6 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        task_pars:
-            Not used in this function.
         workflow_dir:
             Not used in this function.
         workflow_dir_user: