fractal-server 2.0.6__py3-none-any.whl → 2.2.0a0__py3-none-any.whl

fractal_server/app/runner/task_files.py

@@ -1,32 +1,55 @@
 from pathlib import Path
 from typing import Optional
+from typing import Union
+
+from fractal_server.tasks.utils import slugify_task_name
 
 
 def sanitize_component(value: str) -> str:
     """
     Remove {" ", "/", "."} form a string, e.g. going from
     'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
+
+    Args:
+        value: Input strig
     """
     return value.replace(" ", "_").replace("/", "_").replace(".", "_")
 
 
+def task_subfolder_name(order: Union[int, str], task_name: str) -> str:
+    """
+    Get name of task-specific subfolder.
+
+    Args:
+        order:
+        task_name:
+    """
+    task_name_slug = slugify_task_name(task_name)
+    return f"{order}_{task_name_slug}"
+
+
 class TaskFiles:
     """
     Group all file paths pertaining to a task
 
     Attributes:
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+            files. Note: users cannot write directly to this folder.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
+        subfolder_name:
+            Name of task-specific subfolder
+        remote_subfolder:
+            Path to user-side task-specific subfolder
+        task_name:
+            Name of the task
         task_order:
             Positional order of the task within a workflow.
         component:
-            Specific component to run the task for (relevant for tasks that
-            will be executed in parallel over many components).
+            Specific component to run the task for (relevant for tasks to be
+            executed in parallel over many components).
         file_prefix:
             Prefix for all task-related files.
         args:
@@ -39,12 +62,16 @@ class TaskFiles:
             Path for task-execution stderr.
     """
 
-    workflow_dir: Path
-    workflow_dir_user: Path
+    workflow_dir_local: Path
+    workflow_dir_remote: Path
+    remote_subfolder: Path
+    subfolder_name: str
+    task_name: str
     task_order: Optional[int] = None
     component: Optional[str] = None
 
     file_prefix: str
+    file_prefix_with_subfolder: str
     args: Path
     out: Path
     err: Path
@@ -53,14 +80,16 @@ class TaskFiles:
 
     def __init__(
         self,
-        workflow_dir: Path,
-        workflow_dir_user: Path,
+        workflow_dir_local: Path,
+        workflow_dir_remote: Path,
+        task_name: str,
         task_order: Optional[int] = None,
         component: Optional[str] = None,
     ):
-        self.workflow_dir = workflow_dir
-        self.workflow_dir_user = workflow_dir_user
+        self.workflow_dir_local = workflow_dir_local
+        self.workflow_dir_remote = workflow_dir_remote
         self.task_order = task_order
+        self.task_name = task_name
         self.component = component
 
         if self.component is not None:
@@ -72,32 +101,35 @@ class TaskFiles:
         if self.task_order is not None:
             order = str(self.task_order)
         else:
-            order = "task"
+            order = "0"
         self.file_prefix = f"{order}{component_safe}"
-        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
-        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
-        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
-        self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
+        self.subfolder_name = task_subfolder_name(
+            order=order, task_name=self.task_name
+        )
+        self.remote_subfolder = self.workflow_dir_remote / self.subfolder_name
+        self.args = self.remote_subfolder / f"{self.file_prefix}.args.json"
+        self.out = self.remote_subfolder / f"{self.file_prefix}.out"
+        self.err = self.remote_subfolder / f"{self.file_prefix}.err"
+        self.log = self.remote_subfolder / f"{self.file_prefix}.log"
         self.metadiff = (
-            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
+            self.remote_subfolder / f"{self.file_prefix}.metadiff.json"
         )
 
 
 def get_task_file_paths(
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    task_name: str,
     task_order: Optional[int] = None,
     component: Optional[str] = None,
 ) -> TaskFiles:
     """
     Return the corrisponding TaskFiles object
-
-    This function is mainly used as a cache to avoid instantiating needless
-    objects.
     """
     return TaskFiles(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        task_name=task_name,
         task_order=task_order,
         component=component,
     )

fractal_server/app/runner/v1/__init__.py

@@ -33,7 +33,11 @@ from ...models.v1 import WorkflowTask
 from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
+from ..executors.slurm._subprocess_run_as_user import (
+    _mkdir_as_user,
+)
 from ..filenames import WORKFLOW_LOG_FILENAME
+from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
 from ._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
@@ -137,41 +141,48 @@ async def submit_workflow(
         # Define and create server-side working folder
         project_id = workflow.project_id
         timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
-        WORKFLOW_DIR = (
-            settings.FRACTAL_RUNNER_WORKING_BASE_DIR
-            / (
-                f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
-                f"_{timestamp_string}"
-            )
-        ).resolve()
+        WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
+            f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
+            f"_{timestamp_string}"
+        )
 
-        if WORKFLOW_DIR.exists():
-            raise RuntimeError(f"Workflow dir {WORKFLOW_DIR} already exists.")
+        if WORKFLOW_DIR_LOCAL.exists():
+            raise RuntimeError(
+                f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists."
+            )
 
-        # Create WORKFLOW_DIR with 755 permissions
+        # Create WORKFLOW_DIR
         original_umask = os.umask(0)
-        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
+        WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
         os.umask(original_umask)
 
-        # Define and create user-side working folder, if needed
+        # Define and create WORKFLOW_DIR_REMOTE
         if FRACTAL_RUNNER_BACKEND == "local":
-            WORKFLOW_DIR_USER = WORKFLOW_DIR
+            WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
         elif FRACTAL_RUNNER_BACKEND == "slurm":
-
-            from ..executors.slurm._subprocess_run_as_user import (
-                _mkdir_as_user,
+            WORKFLOW_DIR_REMOTE = (
+                Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
             )
+            _mkdir_as_user(folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user)
 
-            WORKFLOW_DIR_USER = (
-                Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
-            ).resolve()
-            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
-        else:
-            raise ValueError(f"{FRACTAL_RUNNER_BACKEND=} not supported")
+        # Create all tasks subfolders
+        for order in range(job.first_task_index, job.last_task_index + 1):
+            subfolder_name = task_subfolder_name(
+                order=order,
+                task_name=workflow.task_list[order].task.name,
+            )
+            original_umask = os.umask(0)
+            (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+            os.umask(original_umask)
+            if FRACTAL_RUNNER_BACKEND == "slurm":
+                _mkdir_as_user(
+                    folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
+                    user=slurm_user,
+                )
 
         # Update db
-        job.working_dir = WORKFLOW_DIR.as_posix()
-        job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+        job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
+        job.working_dir_user = WORKFLOW_DIR_REMOTE.as_posix()
         db_sync.merge(job)
         db_sync.commit()
 
@@ -192,7 +203,7 @@ async def submit_workflow(
 
         # Write logs
         logger_name = f"WF{workflow_id}_job{job_id}"
-        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
             log_file_path=log_file_path,
@@ -239,8 +250,8 @@ async def submit_workflow(
             slurm_user=slurm_user,
             slurm_account=job.slurm_account,
             user_cache_dir=user_cache_dir,
-            workflow_dir=WORKFLOW_DIR,
-            workflow_dir_user=WORKFLOW_DIR_USER,
+            workflow_dir_local=WORKFLOW_DIR_LOCAL,
+            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,

fractal_server/app/runner/v1/_common.py

@@ -36,8 +36,8 @@ from fractal_server.app.runner.task_files import get_task_file_paths
 def no_op_submit_setup_call(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
 ) -> dict:
     """
     Default (no-operation) interface of submit_setup_call.
@@ -113,8 +113,8 @@ def call_single_task(
     *,
     wftask: WorkflowTask,
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
 ) -> TaskParameters:
     """
@@ -133,8 +133,8 @@ def call_single_task(
 
     If the executor then impersonates another user (as in the
     `FractalSlurmExecutor`), this function is run by that user.  For this
-    reason, it should not write any file to workflow_dir, or it may yield
-    permission errors.
+    reason, it should not write any file to `workflow_dir_local`, or it may
+    yield permission errors.
 
     Args:
         wftask:
@@ -143,12 +143,12 @@ def call_single_task(
         task_pars:
             The parameters required to run the task which are not specific to
             the task, e.g., I/O paths.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors). If `None`, it is set to be
-            equal to `workflow_dir`.
+            equal to `workflow_dir_remote`.
         logger_name:
             Name of the logger
 
@@ -164,18 +164,18 @@ def call_single_task(
                             information to the TaskExecutionError, such as task
                             order and name.
         JobExecutionError: If the wrapped task raises a job-related error.
-        RuntimeError: If the `workflow_dir` is falsy.
     """
 
     logger = get_logger(logger_name)
 
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
     )
 
     # write args file (by assembling task_pars and wftask.args)
@@ -248,8 +248,8 @@ def call_single_parallel_task(
     *,
     wftask: WorkflowTask,
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
 ) -> Any:
     """
     Call a single instance of a parallel task
@@ -274,9 +274,9 @@ def call_single_parallel_task(
             The task to execute.
         task_pars:
             The parameters to pass on to the task.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors).
 
@@ -290,17 +290,18 @@ def call_single_parallel_task(
                             information to the TaskExecutionError, such as task
                             order and name.
         JobExecutionError: If the wrapped task raises a job-related error.
-        RuntimeError: If the `workflow_dir` is falsy.
+        RuntimeError: If the `workflow_dir_local` is falsy.
     """
-    if not workflow_dir:
+    if not workflow_dir_local:
         raise RuntimeError
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
         component=component,
     )
 
@@ -363,8 +364,8 @@ def call_parallel_task(
     executor: Executor,
     wftask: WorkflowTask,
     task_pars_depend: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
     logger_name: Optional[str] = None,
 ) -> TaskParameters:
@@ -387,9 +388,9 @@ def call_parallel_task(
             The parallel task to run.
         task_pars_depend:
             The task parameters to be passed on to the parallel task.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors).
         submit_setup_call:
@@ -405,8 +406,8 @@ def call_parallel_task(
     """
     logger = get_logger(logger_name)
 
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     try:
         component_list = task_pars_depend.metadata[
@@ -424,8 +425,8 @@ def call_parallel_task(
     try:
         extra_setup = submit_setup_call(
             wftask=wftask,
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
         )
     except Exception as e:
         tb = "".join(traceback.format_tb(e.__traceback__))
@@ -443,8 +444,8 @@ def call_parallel_task(
         call_single_parallel_task,
         wftask=wftask,
         task_pars=actual_task_pars_depend,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
     )
 
     # Submit tasks for execution. Note that `for _ in map_iter:
@@ -511,16 +512,17 @@ def execute_tasks(
     executor: Executor,
     task_list: list[WorkflowTask],
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
     logger_name: str,
 ) -> TaskParameters:
     """
     Submit a list of WorkflowTasks for execution
 
-    **Note:** At the end of each task, write current metadata to `working_dir /
-    METADATA_FILENAME`, so that they can be read as part of the [`get_job`
+    **Note:** At the end of each task, write current metadata to
+    `workflow_dir_local / METADATA_FILENAME`, so that they can be read as part
+    of the [`get_job`
     endpoint](../../api/v1/job/#fractal_server.app.routes.api.v1.job.get_job).
 
     Arguments:
@@ -531,12 +533,12 @@ def execute_tasks(
             The list of wftasks to be run
         task_pars:
             The task parameters to be passed on to the first task of the list.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors). If `None`, it is set to be
-            equal to `workflow_dir`.
+            equal to `workflow_dir_local`.
         submit_setup_call:
             An optional function that computes configuration parameters for
             the executor.
@@ -548,8 +550,8 @@ def execute_tasks(
             A TaskParameters object which constitutes the output of the last
             task in the list.
     """
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     logger = get_logger(logger_name)
 
@@ -565,8 +567,8 @@ def execute_tasks(
                 executor=executor,
                 wftask=this_wftask,
                 task_pars_depend=current_task_pars,
-                workflow_dir=workflow_dir,
-                workflow_dir_user=workflow_dir_user,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
                 submit_setup_call=submit_setup_call,
                 logger_name=logger_name,
             )
@@ -575,8 +577,8 @@ def execute_tasks(
             try:
                 extra_setup = submit_setup_call(
                     wftask=this_wftask,
-                    workflow_dir=workflow_dir,
-                    workflow_dir_user=workflow_dir_user,
+                    workflow_dir_local=workflow_dir_local,
+                    workflow_dir_remote=workflow_dir_remote,
                 )
             except Exception as e:
                 tb = "".join(traceback.format_tb(e.__traceback__))
@@ -594,8 +596,8 @@ def execute_tasks(
                 call_single_task,
                 wftask=this_wftask,
                 task_pars=current_task_pars,
-                workflow_dir=workflow_dir,
-                workflow_dir_user=workflow_dir_user,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
                 logger_name=logger_name,
                 **extra_setup,
             )
@@ -607,11 +609,11 @@ def execute_tasks(
         )
 
         # Write most recent metadata to METADATA_FILENAME
-        with open(workflow_dir / METADATA_FILENAME, "w") as f:
+        with open(workflow_dir_local / METADATA_FILENAME, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
         # Write most recent metadata to HISTORY_FILENAME
-        with open(workflow_dir / HISTORY_FILENAME, "w") as f:
+        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
             json.dump(current_task_pars.history, f, indent=2)
 
     return current_task_pars

fractal_server/app/runner/v1/_local/__init__.py

@@ -40,7 +40,7 @@ def _process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
+    workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
 ) -> dict[str, Any]:
@@ -66,8 +66,8 @@ def _process_workflow(
                 metadata=input_metadata,
                 history=input_history,
             ),
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
@@ -85,8 +85,8 @@ async def process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     user_cache_dir: Optional[str] = None,
@@ -115,12 +115,13 @@ async def process_workflow(
             Initial metadata, passed to the first task
         logger_name:
             Name of the logger to log information on the run to
-        workflow_dir:
+        workflow_dir_local:
             Working directory for this run.
-        workflow_dir_user:
+        workflow_dir_remote:
             Working directory for this run, on the user side. This argument is
             present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from `workflow_dir`.
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
         slurm_user:
             Username to impersonate to run the workflow. This argument is
             present for compatibility with the standard backend interface, but
@@ -157,10 +158,10 @@ async def process_workflow(
             of the workflow
     """
 
-    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
         raise NotImplementedError(
             "Local backend does not support different directories "
-            f"{workflow_dir=} and {workflow_dir_user=}"
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
         )
 
     # Set values of first_task_index and last_task_index
@@ -178,7 +179,7 @@ async def process_workflow(
         input_metadata=input_metadata,
         input_history=input_history,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
+        workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )

fractal_server/app/runner/v1/_local/_submit_setup.py

@@ -21,8 +21,8 @@ from ._local_config import get_local_backend_config
 def _local_submit_setup(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Optional[Path] = None,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Optional[Path] = None,
+    workflow_dir_remote: Optional[Path] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -31,9 +31,9 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Not used in this function.
-        workflow_dir_user:
+        workflow_dir_remote:
             Not used in this function.
 
     Returns: