fractal-server 2.1.0__py3-none-any.whl → 2.2.0a1__py3-none-any.whl

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -50,8 +50,8 @@ def _process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     first_task_index: int,
     last_task_index: int,
     slurm_user: Optional[str] = None,
@@ -86,8 +86,8 @@ def _process_workflow(
         keep_logs=True,
         slurm_user=slurm_user,
         user_cache_dir=user_cache_dir,
-        working_dir=workflow_dir,
-        working_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
@@ -102,8 +102,8 @@ def _process_workflow(
                 metadata=input_metadata,
                 history=input_history,
             ),
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
             submit_setup_call=_slurm_submit_setup,
             logger_name=logger_name,
         )
@@ -121,8 +121,8 @@ async def process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
@@ -152,8 +152,8 @@ async def process_workflow(
         input_metadata=input_metadata,
         input_history=input_history,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         slurm_user=slurm_user,
         slurm_account=slurm_account,
         user_cache_dir=user_cache_dir,
@@ -166,8 +166,8 @@ async def process_workflow(
 
 def get_slurm_config(
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     config_path: Optional[Path] = None,
 ) -> SlurmConfig:
     """
@@ -187,13 +187,13 @@ def get_slurm_config(
         wftask:
             WorkflowTask for which the SLURM configuration is is to be
             prepared.
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
         config_path:
             Path of aFractal  SLURM configuration file; if `None`, use
             `FRACTAL_SLURM_CONFIG_FILE` variable from settings.

fractal_server/app/runner/v1/_slurm/_submit_setup.py

@@ -24,8 +24,8 @@ from fractal_server.app.models.v1 import WorkflowTask
 def _slurm_submit_setup(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -43,13 +43,13 @@ def _slurm_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
 
     Returns:
         submit_setup_dict:
@@ -61,15 +61,16 @@ def _slurm_submit_setup(
     # Get SlurmConfig object
     slurm_config = get_slurm_config(
         wftask=wftask,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
     )
 
     # Get TaskFiles object
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
     )
 
     # Prepare and return output dictionary

fractal_server/app/runner/v1/_slurm/get_slurm_config.py

@@ -17,8 +17,8 @@ from fractal_server.app.runner.executors.slurm._slurm_config import (
 
 def get_slurm_config(
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     config_path: Optional[Path] = None,
 ) -> SlurmConfig:
     """
@@ -38,13 +38,13 @@ def get_slurm_config(
         wftask:
             WorkflowTask for which the SLURM configuration is is to be
             prepared.
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
         config_path:
             Path of aFractal  SLURM configuration file; if `None`, use
             `FRACTAL_SLURM_CONFIG_FILE` variable from settings.

fractal_server/app/runner/v2/__init__.py

@@ -25,8 +25,13 @@ from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
+from ..executors.slurm._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
+from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
+from ._local_experimental import (
+    process_workflow as local_experimental_process_workflow,
+)
 from ._slurm import process_workflow as slurm_process_workflow
 from .handle_failed_job import assemble_filters_failed_job
 from .handle_failed_job import assemble_history_failed_job
@@ -35,6 +40,7 @@ from fractal_server import __VERSION__
 
 _backends = {}
 _backends["local"] = local_process_workflow
+_backends["local_experimental"] = local_experimental_process_workflow
 _backends["slurm"] = slurm_process_workflow
 
 
@@ -78,6 +84,8 @@ async def submit_workflow(
     FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
     if FRACTAL_RUNNER_BACKEND == "local":
         process_workflow = local_process_workflow
+    elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+        process_workflow = local_experimental_process_workflow
     elif FRACTAL_RUNNER_BACKEND == "slurm":
         process_workflow = slurm_process_workflow
     else:
@@ -108,29 +116,67 @@ async def submit_workflow(
             return
 
         # Define and create server-side working folder
-        WORKFLOW_DIR = Path(job.working_dir)
-        if WORKFLOW_DIR.exists():
+        WORKFLOW_DIR_LOCAL = Path(job.working_dir)
+        if WORKFLOW_DIR_LOCAL.exists():
             job.status = JobStatusTypeV2.FAILED
             job.end_timestamp = get_timestamp()
-            job.log = f"Workflow dir {WORKFLOW_DIR} already exists."
+            job.log = f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
             db_sync.merge(job)
             db_sync.commit()
             db_sync.close()
             return
 
-        # Create WORKFLOW_DIR with 755 permissions
-        original_umask = os.umask(0)
-        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
-        os.umask(original_umask)
+        try:
 
-        # Define and create user-side working folder, if needed
-        WORKFLOW_DIR_USER = Path(job.working_dir_user)
-        if FRACTAL_RUNNER_BACKEND == "slurm":
-            from ..executors.slurm._subprocess_run_as_user import (
-                _mkdir_as_user,
-            )
+            # Create WORKFLOW_DIR
+            original_umask = os.umask(0)
+            WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
+
+            os.umask(original_umask)
 
-            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
+            # Define and create WORKFLOW_DIR_REMOTE
+            if FRACTAL_RUNNER_BACKEND == "local":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "local_experimental":
+                WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
+            elif FRACTAL_RUNNER_BACKEND == "slurm":
+                WORKFLOW_DIR_REMOTE = (
+                    Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
+                )
+                _mkdir_as_user(
+                    folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user
+                )
+
+            # Create all tasks subfolders
+            for order in range(job.first_task_index, job.last_task_index + 1):
+                this_wftask = workflow.task_list[order]
+                if this_wftask.is_legacy_task:
+                    task_name = this_wftask.task_legacy.name
+                else:
+                    task_name = this_wftask.task.name
+                subfolder_name = task_subfolder_name(
+                    order=order,
+                    task_name=task_name,
+                )
+                original_umask = os.umask(0)
+                (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+                os.umask(original_umask)
+                if FRACTAL_RUNNER_BACKEND == "slurm":
+                    _mkdir_as_user(
+                        folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
+                        user=slurm_user,
+                    )
+        except Exception as e:
+            job.status = JobStatusTypeV2.FAILED
+            job.end_timestamp = get_timestamp()
+            job.log = (
+                "An error occurred while creating job folder and subfolders.\n"
+                f"Original error: {str(e)}"
+            )
+            db_sync.merge(job)
+            db_sync.commit()
+            db_sync.close()
+            return
 
         # After Session.commit() is called, either explicitly or when using a
         # context manager, all objects associated with the Session are expired.
@@ -145,10 +191,12 @@ async def submit_workflow(
 
         db_sync.refresh(dataset)
         db_sync.refresh(workflow)
+        for wftask in workflow.task_list:
+            db_sync.refresh(wftask)
 
         # Write logs
         logger_name = f"WF{workflow_id}_job{job_id}"
-        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
             log_file_path=log_file_path,
@@ -189,8 +237,8 @@ async def submit_workflow(
             slurm_user=slurm_user,
             slurm_account=job.slurm_account,
             user_cache_dir=user_cache_dir,
-            workflow_dir=WORKFLOW_DIR,
-            workflow_dir_user=WORKFLOW_DIR_USER,
+            workflow_dir_local=WORKFLOW_DIR_LOCAL,
+            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,

fractal_server/app/runner/v2/_local/__init__.py

@@ -36,7 +36,7 @@ def _process_workflow(
     workflow: WorkflowV2,
     dataset: DatasetV2,
     logger_name: str,
-    workflow_dir: Path,
+    workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
 ) -> dict:
@@ -57,8 +57,8 @@ def _process_workflow(
             ],  # noqa
             dataset=dataset,
             executor=executor,
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
@@ -69,8 +69,8 @@ async def process_workflow(
     *,
     workflow: WorkflowV2,
     dataset: DatasetV2,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
@@ -94,12 +94,13 @@ async def process_workflow(
             The workflow to be run
         dataset:
             Initial dataset.
-        workflow_dir:
+        workflow_dir_local:
             Working directory for this run.
-        workflow_dir_user:
+        workflow_dir_remote:
             Working directory for this run, on the user side. This argument is
             present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from `workflow_dir`.
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
         first_task_index:
             Positional index of the first task to execute; if `None`, start
             from `0`.
@@ -137,10 +138,10 @@ async def process_workflow(
             of the workflow
     """
 
-    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
         raise NotImplementedError(
             "Local backend does not support different directories "
-            f"{workflow_dir=} and {workflow_dir_user=}"
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
         )
 
     # Set values of first_task_index and last_task_index
@@ -155,7 +156,7 @@ async def process_workflow(
         workflow=workflow,
         dataset=dataset,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
+        workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )

fractal_server/app/runner/v2/_local/_local_config.py

@@ -96,7 +96,7 @@ def get_local_backend_config(
         )
 
     if wftask_meta and key in wftask_meta:
-        parallel_tasks_per_job = wftask.meta[key]
+        parallel_tasks_per_job = wftask_meta[key]
     else:
         if not config_path:
             settings = Inject(get_settings)

fractal_server/app/runner/v2/_local/_submit_setup.py

@@ -22,8 +22,8 @@ from ._local_config import get_local_backend_config
 def _local_submit_setup(
     *,
     wftask: WorkflowTaskV2,
-    workflow_dir: Optional[Path] = None,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Optional[Path] = None,
+    workflow_dir_remote: Optional[Path] = None,
     which_type: Literal["non_parallel", "parallel"],
 ) -> dict[str, object]:
     """
@@ -33,9 +33,9 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Not used in this function.
-        workflow_dir_user:
+        workflow_dir_remote:
             Not used in this function.
 
     Returns:

fractal_server/app/runner/v2/_local_experimental/__init__.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from typing import Optional
+
+from ....models.v2 import DatasetV2
+from ....models.v2 import WorkflowV2
+from ...async_wrap import async_wrap
+from ...filenames import SHUTDOWN_FILENAME
+from ...set_start_and_last_task_index import set_start_and_last_task_index
+from ..runner import execute_tasks_v2
+from ._submit_setup import _local_submit_setup
+from .executor import FractalProcessPoolExecutor
+
+
+def _process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    logger_name: str,
+    workflow_dir_local: Path,
+    first_task_index: int,
+    last_task_index: int,
+) -> dict:
+    """
+    Internal processing routine
+
+    Schedules the workflow using a `FractalProcessPoolExecutor`.
+
+    Cf.
+    [process_workflow][fractal_server.app.runner.v2._local_experimental.process_workflow]
+    for the call signature.
+    """
+
+    with FractalProcessPoolExecutor(
+        shutdown_file=workflow_dir_local / SHUTDOWN_FILENAME
+    ) as executor:
+        new_dataset_attributes = execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)  # noqa
+            ],  # noqa
+            dataset=dataset,
+            executor=executor,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+        )
+    return new_dataset_attributes
+
+
+async def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    # Slurm-specific
+    user_cache_dir: Optional[str] = None,
+    slurm_user: Optional[str] = None,
+    slurm_account: Optional[str] = None,
+    worker_init: Optional[str] = None,
+) -> dict:
+    """
+    Run a workflow
+
+    This function is responsible for running a workflow on some input data,
+    saving the output and taking care of any exception raised during the run.
+
+    NOTE: This is the `local_experimental` backend's public interface,
+    which also works as a reference implementation for other backends.
+
+    Args:
+        workflow:
+            The workflow to be run
+        dataset:
+            Initial dataset.
+        workflow_dir_local:
+            Working directory for this run.
+        workflow_dir_remote:
+            Working directory for this run, on the user side. This argument is
+            present for compatibility with the standard backend interface, but
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+        logger_name: Logger name
+        slurm_user:
+            Username to impersonate to run the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        slurm_account:
+            SLURM account to use when running the workflow. This argument is
+            present for compatibility with the standard backend interface, but
+            is ignored in the `local` backend.
+        user_cache_dir:
+            Cache directory of the user who will run the workflow. This
+            argument is present for compatibility with the standard backend
+            interface, but is ignored in the `local` backend.
+        worker_init:
+            Any additional, usually backend specific, information to be passed
+            to the backend executor. This argument is present for compatibility
+            with the standard backend interface, but is ignored in the `local`
+            backend.
+
+    Raises:
+        TaskExecutionError: wrapper for errors raised during tasks' execution
+                            (positive exit codes).
+        JobExecutionError: wrapper for errors raised by the tasks' executors
+                           (negative exit codes).
+
+    Returns:
+        output_dataset_metadata:
+            The updated metadata for the dataset, as returned by the last task
+            of the workflow
+    """
+
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
+        raise NotImplementedError(
+            "LocalExperimental backend does not support different directories "
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    new_dataset_attributes = await async_wrap(_process_workflow)(
+        workflow=workflow,
+        dataset=dataset,
+        logger_name=logger_name,
+        workflow_dir_local=workflow_dir_local,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+    return new_dataset_attributes

fractal_server/app/runner/v2/_local_experimental/_local_config.py

@@ -0,0 +1,108 @@
+"""
+Submodule to handle the local-backend configuration for a WorkflowTask
+"""
+import json
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic.error_wrappers import ValidationError
+
+from .....config import get_settings
+from .....syringe import Inject
+from ....models.v2 import WorkflowTaskV2
+
+
+class LocalBackendConfigError(ValueError):
+    """
+    Local-backend configuration error
+    """
+
+    pass
+
+
+class LocalBackendConfig(BaseModel, extra=Extra.forbid):
+    """
+    Specifications of the local-backend configuration
+
+    Attributes:
+        parallel_tasks_per_job:
+            Maximum number of tasks to be run in parallel as part of a call to
+            `FractalProcessPoolExecutor.map`; if `None`, then all tasks will
+            start at the same time.
+    """
+
+    parallel_tasks_per_job: Optional[int]
+
+
+def get_default_local_backend_config():
+    """
+    Return a default `LocalBackendConfig` configuration object
+    """
+    return LocalBackendConfig(parallel_tasks_per_job=None)
+
+
+def get_local_backend_config(
+    wftask: WorkflowTaskV2,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+) -> LocalBackendConfig:
+    """
+    Prepare a `LocalBackendConfig` configuration object
+
+    The sources for `parallel_tasks_per_job` attributes, starting from the
+    highest-priority one, are
+
+    1. Properties in `wftask.meta_parallel` or `wftask.meta_non_parallel`
+       (depending on `which_type`);
+    2. The general content of the local-backend configuration file;
+    3. The default value (`None`).
+
+    Arguments:
+        wftask:
+            WorkflowTaskV2 for which the backend configuration should
+            be prepared.
+        config_path:
+            Path of local-backend configuration file; if `None`, use
+            `FRACTAL_LOCAL_CONFIG_FILE` variable from settings.
+
+    Returns:
+        A local-backend configuration object
+    """
+
+    key = "parallel_tasks_per_job"
+    default_value = None
+
+    if which_type == "non_parallel":
+        wftask_meta = wftask.meta_non_parallel
+    elif which_type == "parallel":
+        wftask_meta = wftask.meta_parallel
+    else:
+        raise ValueError(
+            "`get_local_backend_config` received an invalid argument"
+            f" {which_type=}."
+        )
+
+    if wftask_meta and key in wftask_meta:
+        parallel_tasks_per_job = wftask_meta[key]
+    else:
+        if not config_path:
+            settings = Inject(get_settings)
+            config_path = settings.FRACTAL_LOCAL_CONFIG_FILE
+        if config_path is None:
+            parallel_tasks_per_job = default_value
+        else:
+            with config_path.open("r") as f:
+                env = json.load(f)
+            try:
+                _ = LocalBackendConfig(**env)
+            except ValidationError as e:
+                raise LocalBackendConfigError(
+                    f"Error while loading {config_path=}. "
+                    f"Original error:\n{str(e)}"
+                )
+
+            parallel_tasks_per_job = env.get(key, default_value)
+    return LocalBackendConfig(parallel_tasks_per_job=parallel_tasks_per_job)