fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/runner/v1/_slurm/get_slurm_config.py

@@ -0,0 +1,163 @@
+from pathlib import Path
+from typing import Optional
+
+from fractal_server.app.models.v1 import WorkflowTask
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    _parse_mem_value,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    load_slurm_config_file,
+)
+from fractal_server.app.runner.executors.slurm._slurm_config import logger
+from fractal_server.app.runner.executors.slurm._slurm_config import SlurmConfig
+from fractal_server.app.runner.executors.slurm._slurm_config import (
+    SlurmConfigError,
+)
+
+
+def get_slurm_config(
+    wftask: WorkflowTask,
+    workflow_dir: Path,
+    workflow_dir_user: Path,
+    config_path: Optional[Path] = None,
+) -> SlurmConfig:
+    """
+    Prepare a `SlurmConfig` configuration object
+
+    The sources for `SlurmConfig` attributes, in increasing priority order, are
+
+    1. The general content of the Fractal SLURM configuration file.
+    2. The GPU-specific content of the Fractal SLURM configuration file, if
+        appropriate.
+    3. Properties in `wftask.meta` (which, for `WorkflowTask`s added through
+       `Workflow.insert_task`, also includes `wftask.task.meta`);
+
+    Note: `wftask.meta` may be `None`.
+
+    Arguments:
+        wftask:
+            WorkflowTask for which the SLURM configuration is is to be
+            prepared.
+        workflow_dir:
+            Server-owned directory to store all task-execution-related relevant
+            files (inputs, outputs, errors, and all meta files related to the
+            job execution). Note: users cannot write directly to this folder.
+        workflow_dir_user:
+            User-side directory with the same scope as `workflow_dir`, and
+            where a user can write.
+        config_path:
+            Path of aFractal  SLURM configuration file; if `None`, use
+            `FRACTAL_SLURM_CONFIG_FILE` variable from settings.
+
+    Returns:
+        slurm_config:
+            The SlurmConfig object
+    """
+
+    logger.debug(
+        "[get_slurm_config] WorkflowTask meta attribute: {wftask.meta=}"
+    )
+
+    # Incorporate slurm_env.default_slurm_config
+    slurm_env = load_slurm_config_file(config_path=config_path)
+    slurm_dict = slurm_env.default_slurm_config.dict(
+        exclude_unset=True, exclude={"mem"}
+    )
+    if slurm_env.default_slurm_config.mem:
+        slurm_dict["mem_per_task_MB"] = slurm_env.default_slurm_config.mem
+
+    # Incorporate slurm_env.batching_config
+    for key, value in slurm_env.batching_config.dict().items():
+        slurm_dict[key] = value
+
+    # Incorporate slurm_env.user_local_exports
+    slurm_dict["user_local_exports"] = slurm_env.user_local_exports
+
+    logger.debug(
+        "[get_slurm_config] Fractal SLURM configuration file: "
+        f"{slurm_env.dict()=}"
+    )
+
+    # GPU-related options
+    # Notes about priority:
+    # 1. This block of definitions takes priority over other definitions from
+    #    slurm_env which are not under the `needs_gpu` subgroup
+    # 2. This block of definitions has lower priority than whatever comes next
+    #    (i.e. from WorkflowTask.meta).
+    if wftask.meta is not None:
+        needs_gpu = wftask.meta.get("needs_gpu", False)
+    else:
+        needs_gpu = False
+    logger.debug(f"[get_slurm_config] {needs_gpu=}")
+    if needs_gpu:
+        for key, value in slurm_env.gpu_slurm_config.dict(
+            exclude_unset=True, exclude={"mem"}
+        ).items():
+            slurm_dict[key] = value
+        if slurm_env.gpu_slurm_config.mem:
+            slurm_dict["mem_per_task_MB"] = slurm_env.gpu_slurm_config.mem
+
+    # Number of CPUs per task, for multithreading
+    if wftask.meta is not None and "cpus_per_task" in wftask.meta:
+        cpus_per_task = int(wftask.meta["cpus_per_task"])
+        slurm_dict["cpus_per_task"] = cpus_per_task
+
+    # Required memory per task, in MB
+    if wftask.meta is not None and "mem" in wftask.meta:
+        raw_mem = wftask.meta["mem"]
+        mem_per_task_MB = _parse_mem_value(raw_mem)
+        slurm_dict["mem_per_task_MB"] = mem_per_task_MB
+
+    # Job name
+    job_name = wftask.task.name.replace(" ", "_")
+    slurm_dict["job_name"] = job_name
+
+    # Optional SLURM arguments and extra lines
+    if wftask.meta is not None:
+        account = wftask.meta.get("account", None)
+        if account is not None:
+            error_msg = (
+                f"Invalid {account=} property in WorkflowTask `meta` "
+                "attribute.\n"
+                "SLURM account must be set in the request body of the "
+                "apply-workflow endpoint, or by modifying the user properties."
+            )
+            logger.error(error_msg)
+            raise SlurmConfigError(error_msg)
+        for key in ["time", "gres", "constraint"]:
+            value = wftask.meta.get(key, None)
+            if value:
+                slurm_dict[key] = value
+    if wftask.meta is not None:
+        extra_lines = wftask.meta.get("extra_lines", [])
+    else:
+        extra_lines = []
+    extra_lines = slurm_dict.get("extra_lines", []) + extra_lines
+    if len(set(extra_lines)) != len(extra_lines):
+        logger.debug(
+            "[get_slurm_config] Removing repeated elements "
+            f"from {extra_lines=}."
+        )
+        extra_lines = list(set(extra_lines))
+    slurm_dict["extra_lines"] = extra_lines
+
+    # Job-batching parameters (if None, they will be determined heuristically)
+    if wftask.meta is not None:
+        tasks_per_job = wftask.meta.get("tasks_per_job", None)
+        parallel_tasks_per_job = wftask.meta.get(
+            "parallel_tasks_per_job", None
+        )
+    else:
+        tasks_per_job = None
+        parallel_tasks_per_job = None
+    slurm_dict["tasks_per_job"] = tasks_per_job
+    slurm_dict["parallel_tasks_per_job"] = parallel_tasks_per_job
+
+    # Put everything together
+    logger.debug(
+        "[get_slurm_config] Now create a SlurmConfig object based "
+        f"on {slurm_dict=}"
+    )
+    slurm_config = SlurmConfig(**slurm_dict)
+
+    return slurm_config

fractal_server/app/runner/v1/common.py

@@ -0,0 +1,117 @@
+"""
+Common utilities and routines for runner backends (public API)
+
+This module includes utilities and routines that are of use to implement
+runner backends but that should also be exposed to the other components of
+`Fractal Server`.
+"""
+import json
+from json import JSONEncoder
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from ....logger import close_logger as close_job_logger  # noqa F401
+from ...models.v1 import Dataset
+from ...models.v1 import Workflow
+
+
+class TaskParameterEncoder(JSONEncoder):
+    """
+    Convenience JSONEncoder that serialises `Path`s as strings
+    """
+
+    def default(self, value):
+        if isinstance(value, Path):
+            return value.as_posix()
+        return JSONEncoder.default(self, value)
+
+
+class TaskParameters(BaseModel):
+    """
+    Wrapper for task input parameters
+
+    Instances of this class are used to pass parameters from the output of a
+    task to the input of the next one.
+
+    Attributes:
+        input_paths:
+            Input paths as derived by the input dataset.
+        output_paths:
+            Output path as derived from the output dataset.
+        metadata:
+            Dataset metadata, as found in the input dataset or as updated by
+            the previous task.
+        history:
+            Dataset history, as found in the input dataset or as updated by
+            the previous task.
+    """
+
+    input_paths: list[Path]
+    output_path: Path
+    metadata: dict[str, Any]
+    history: list[dict[str, Any]]
+
+    class Config:
+        arbitrary_types_allowed = True
+        extra = "forbid"
+
+
+def validate_workflow_compatibility(
+    *,
+    input_dataset: Dataset,
+    workflow: Workflow,
+    output_dataset: Dataset,
+    first_task_index: int,
+    last_task_index: int,
+) -> None:
+    """
+    Check compatibility of workflow and input / ouptut dataset
+    """
+    # Check input_dataset type
+    workflow_input_type = workflow.task_list[first_task_index].task.input_type
+    if (
+        workflow_input_type != "Any"
+        and workflow_input_type != input_dataset.type
+    ):
+        raise TypeError(
+            f"Incompatible types `{workflow_input_type}` of workflow "
+            f"`{workflow.name}` and `{input_dataset.type}` of dataset "
+            f"`{input_dataset.name}`"
+        )
+
+    # Check output_dataset type
+    workflow_output_type = workflow.task_list[last_task_index].task.output_type
+    if (
+        workflow_output_type != "Any"
+        and workflow_output_type != output_dataset.type
+    ):
+        raise TypeError(
+            f"Incompatible types `{workflow_output_type}` of workflow "
+            f"`{workflow.name}` and `{output_dataset.type}` of dataset "
+            f"`{output_dataset.name}`"
+        )
+
+
+def write_args_file(
+    *args: dict[str, Any],
+    path: Path,
+):
+    """
+    Merge arbitrary dictionaries and write to file
+
+    Args:
+        *args:
+            One or more dictionaries that will be merged into one respecting
+            the order with which they are passed in, i.e., last in overrides
+            previous ones.
+        path:
+            Destination for serialised file.
+    """
+    out = {}
+    for d in args:
+        out.update(d)
+
+    with open(path, "w") as f:
+        json.dump(out, f, cls=TaskParameterEncoder, indent=4)

fractal_server/app/runner/{handle_failed_job.py → v1/handle_failed_job.py}

@@ -19,13 +19,13 @@ from pathlib import Path
 from typing import Any
 from typing import Optional
 
-from ..models import ApplyWorkflow
-from ..models import Dataset
-from ..models import Workflow
-from ..models import WorkflowTask
-from ..models import WorkflowTaskStatusType
-from ._common import HISTORY_FILENAME
-from ._common import METADATA_FILENAME
+from ...models.v1 import ApplyWorkflow
+from ...models.v1 import Dataset
+from ...models.v1 import Workflow
+from ...models.v1 import WorkflowTask
+from ...schemas.v1 import WorkflowTaskStatusTypeV1
+from ..filenames import HISTORY_FILENAME
+from ..filenames import METADATA_FILENAME
 
 
 def assemble_history_failed_job(
@@ -98,7 +98,7 @@ def assemble_history_failed_job(
         failed_wftask_dump["task"] = failed_wftask.task.model_dump()
         new_history_item = dict(
             workflowtask=failed_wftask_dump,
-            status=WorkflowTaskStatusType.FAILED,
+            status=WorkflowTaskStatusTypeV1.FAILED,
             parallelization=dict(
                 parallelization_level=failed_wftask.parallelization_level,
             ),

fractal_server/app/runner/v2/__init__.py

@@ -0,0 +1,337 @@
+"""
+Runner backend subsystem root V2
+
+This module is the single entry point to the runner backend subsystem V2.
+Other subystems should only import this module and not its submodules or
+the individual backends.
+"""
+import os
+import traceback
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy.orm.attributes import flag_modified
+
+from ....config import get_settings
+from ....logger import close_logger
+from ....logger import set_logger
+from ....syringe import Inject
+from ....utils import get_timestamp
+from ...db import DB
+from ...models.v2 import DatasetV2
+from ...models.v2 import JobV2
+from ...models.v2 import WorkflowTaskV2
+from ...models.v2 import WorkflowV2
+from ...schemas.v2 import JobStatusTypeV2
+from ..exceptions import JobExecutionError
+from ..exceptions import TaskExecutionError
+from ..filenames import WORKFLOW_LOG_FILENAME
+from ._local import process_workflow as local_process_workflow
+from ._slurm import process_workflow as slurm_process_workflow
+from .handle_failed_job import assemble_filters_failed_job
+from .handle_failed_job import assemble_history_failed_job
+from .handle_failed_job import assemble_images_failed_job
+from .runner import execute_tasks_v2  # noqa
+from fractal_server import __VERSION__
+
+_backends = {}
+_backends["local"] = local_process_workflow
+_backends["slurm"] = slurm_process_workflow
+
+
+async def submit_workflow(
+    *,
+    workflow_id: int,
+    dataset_id: int,
+    job_id: int,
+    worker_init: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    user_cache_dir: Optional[str] = None,
+) -> None:
+    """
+    Prepares a workflow and applies it to a dataset
+
+    This function wraps the process_workflow one, which is different for each
+    backend (e.g. local or slurm backend).
+
+    Args:
+        workflow_id:
+            ID of the workflow being applied
+        dataset_id:
+            Dataset ID
+        job_id:
+            Id of the job record which stores the state for the current
+            workflow application.
+        worker_init:
+            Custom executor parameters that get parsed before the execution of
+            each task.
+        user_cache_dir:
+            Cache directory (namely a path where the user can write); for the
+            slurm backend, this is used as a base directory for
+            `job.working_dir_user`.
+        slurm_user:
+            The username to impersonate for the workflow execution, for the
+            slurm backend.
+    """
+
+    # Declare runner backend and set `process_workflow` function
+    settings = Inject(get_settings)
+    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+    if FRACTAL_RUNNER_BACKEND == "local":
+        process_workflow = local_process_workflow
+    elif FRACTAL_RUNNER_BACKEND == "slurm":
+        process_workflow = slurm_process_workflow
+    else:
+        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+
+    with next(DB.get_sync_db()) as db_sync:
+
+        job: JobV2 = db_sync.get(JobV2, job_id)
+        if not job:
+            raise ValueError(f"Cannot fetch job {job_id} from database")
+
+        dataset: DatasetV2 = db_sync.get(DatasetV2, dataset_id)
+        workflow: WorkflowV2 = db_sync.get(WorkflowV2, workflow_id)
+        if not (dataset and workflow):
+            log_msg = ""
+            if not dataset:
+                log_msg += f"Cannot fetch dataset {dataset_id} from database\n"
+            if not workflow:
+                log_msg += (
+                    f"Cannot fetch workflow {workflow_id} from database\n"
+                )
+            job.status = JobStatusTypeV2.FAILED
+            job.end_timestamp = get_timestamp()
+            job.log = log_msg
+            db_sync.merge(job)
+            db_sync.commit()
+            db_sync.close()
+            return
+
+        # Define and create server-side working folder
+        project_id = workflow.project_id
+        timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
+        WORKFLOW_DIR = (
+            settings.FRACTAL_RUNNER_WORKING_BASE_DIR
+            / (
+                f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
+                f"_{timestamp_string}"
+            )
+        ).resolve()
+
+        if WORKFLOW_DIR.exists():
+            raise RuntimeError(f"Workflow dir {WORKFLOW_DIR} already exists.")
+
+        # Create WORKFLOW_DIR with 755 permissions
+        original_umask = os.umask(0)
+        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
+        os.umask(original_umask)
+
+        # Define and create user-side working folder, if needed
+        if FRACTAL_RUNNER_BACKEND == "local":
+            WORKFLOW_DIR_USER = WORKFLOW_DIR
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+
+            from ..executors.slurm._subprocess_run_as_user import (
+                _mkdir_as_user,
+            )
+
+            WORKFLOW_DIR_USER = (
+                Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
+            ).resolve()
+            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
+        else:
+            raise ValueError(f"{FRACTAL_RUNNER_BACKEND=} not supported")
+
+        # Update db
+        job.working_dir = WORKFLOW_DIR.as_posix()
+        job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+        db_sync.merge(job)
+        db_sync.commit()
+
+        # After Session.commit() is called, either explicitly or when using a
+        # context manager, all objects associated with the Session are expired.
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_basics.html#opening-and-closing-a-session
+        # https://docs.sqlalchemy.org/en/14/orm/
+        #   session_state_management.html#refreshing-expiring
+
+        # See issue #928:
+        # https://github.com/fractal-analytics-platform/
+        #   fractal-server/issues/928
+
+        db_sync.refresh(dataset)
+        db_sync.refresh(workflow)
+
+        # Write logs
+        logger_name = f"WF{workflow_id}_job{job_id}"
+        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        logger = set_logger(
+            logger_name=logger_name,
+            log_file_path=log_file_path,
+        )
+        logger.info(
+            f'Start execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f"fractal_server.__VERSION__: {__VERSION__}")
+        logger.debug(f"FRACTAL_RUNNER_BACKEND: {FRACTAL_RUNNER_BACKEND}")
+        logger.debug(f"slurm_user: {slurm_user}")
+        logger.debug(f"slurm_account: {job.slurm_account}")
+        logger.debug(f"worker_init: {worker_init}")
+        logger.debug(f"job.id: {job.id}")
+        logger.debug(f"job.working_dir: {job.working_dir}")
+        logger.debug(f"job.working_dir_user: {job.working_dir_user}")
+        logger.debug(f"job.first_task_index: {job.first_task_index}")
+        logger.debug(f"job.last_task_index: {job.last_task_index}")
+        logger.debug(f'START workflow "{workflow.name}"')
+
+    try:
+        # "The Session.close() method does not prevent the Session from being
+        # used again. The Session itself does not actually have a distinct
+        # “closed” state; it merely means the Session will release all database
+        # connections and ORM objects."
+        # (https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session.close).
+        #
+        # We close the session before the (possibly long) process_workflow
+        # call, to make sure all DB connections are released. The reason why we
+        # are not using a context manager within the try block is that we also
+        # need access to db_sync in the except branches.
+        db_sync = next(DB.get_sync_db())
+        db_sync.close()
+
+        new_dataset_attributes = await process_workflow(
+            workflow=workflow,
+            dataset=dataset,
+            slurm_user=slurm_user,
+            slurm_account=job.slurm_account,
+            user_cache_dir=user_cache_dir,
+            workflow_dir=WORKFLOW_DIR,
+            workflow_dir_user=WORKFLOW_DIR_USER,
+            logger_name=logger_name,
+            worker_init=worker_init,
+            first_task_index=job.first_task_index,
+            last_task_index=job.last_task_index,
+        )
+
+        logger.info(
+            f'End execution of workflow "{workflow.name}"; '
+            f"more logs at {str(log_file_path)}"
+        )
+        logger.debug(f'END workflow "{workflow.name}"')
+
+        # Update dataset attributes, in case of successful execution
+        dataset.history.extend(new_dataset_attributes["history"])
+        dataset.filters = new_dataset_attributes["filters"]
+        dataset.images = new_dataset_attributes["images"]
+        for attribute_name in ["filters", "history", "images"]:
+            flag_modified(dataset, attribute_name)
+        db_sync.merge(dataset)
+
+        # Update job DB entry
+        job.status = JobStatusTypeV2.DONE
+        job.end_timestamp = get_timestamp()
+        with log_file_path.open("r") as f:
+            logs = f.read()
+        job.log = logs
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except TaskExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        failed_wftask = db_sync.get(WorkflowTaskV2, e.workflow_task_id)
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+            failed_wftask=failed_wftask,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+
+        exception_args_string = "\n".join(e.args)
+        job.log = (
+            f"TASK ERROR: "
+            f"Task name: {e.task_name}, "
+            f"position in Workflow: {e.workflow_task_order}\n"
+            f"TRACEBACK:\n{exception_args_string}"
+        )
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except JobExecutionError as e:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
+        logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+        error = e.assemble_error()
+        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
+        db_sync.merge(job)
+        db_sync.commit()
+
+    except Exception:
+
+        logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
+        logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
+
+        current_traceback = traceback.format_exc()
+
+        # Read dataset attributes produced by the last successful task, and
+        # update the DB dataset accordingly
+        dataset.history = assemble_history_failed_job(
+            job,
+            dataset,
+            workflow,
+            logger,
+        )
+        latest_filters = assemble_filters_failed_job(job)
+        if latest_filters is not None:
+            dataset.filters = latest_filters
+        latest_images = assemble_images_failed_job(job)
+        if latest_images is not None:
+            dataset.images = latest_images
+        db_sync.merge(dataset)
+
+        job.status = JobStatusTypeV2.FAILED
+        job.end_timestamp = get_timestamp()
+        job.log = (
+            f"UNKNOWN ERROR in Fractal job {job.id}\n"
+            f"TRACEBACK:\n{current_traceback}"
+        )
+        db_sync.merge(job)
+        db_sync.commit()
+    finally:
+        close_logger(logger)
+        db_sync.close()