fractal-server 2.0.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

fractal_server/app/runner/v1/_common.py

@@ -36,8 +36,8 @@ from fractal_server.app.runner.task_files import get_task_file_paths
 def no_op_submit_setup_call(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
 ) -> dict:
     """
     Default (no-operation) interface of submit_setup_call.
@@ -113,8 +113,8 @@ def call_single_task(
     *,
     wftask: WorkflowTask,
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
 ) -> TaskParameters:
     """
@@ -133,8 +133,8 @@ def call_single_task(
 
     If the executor then impersonates another user (as in the
     `FractalSlurmExecutor`), this function is run by that user.  For this
-    reason, it should not write any file to workflow_dir, or it may yield
-    permission errors.
+    reason, it should not write any file to `workflow_dir_local`, or it may
+    yield permission errors.
 
     Args:
         wftask:
@@ -143,12 +143,12 @@ def call_single_task(
         task_pars:
             The parameters required to run the task which are not specific to
             the task, e.g., I/O paths.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors). If `None`, it is set to be
-            equal to `workflow_dir`.
+            equal to `workflow_dir_remote`.
         logger_name:
             Name of the logger
 
@@ -164,18 +164,18 @@ def call_single_task(
                             information to the TaskExecutionError, such as task
                             order and name.
         JobExecutionError: If the wrapped task raises a job-related error.
-        RuntimeError: If the `workflow_dir` is falsy.
     """
 
     logger = get_logger(logger_name)
 
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
     )
 
     # write args file (by assembling task_pars and wftask.args)
@@ -248,8 +248,8 @@ def call_single_parallel_task(
     *,
     wftask: WorkflowTask,
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
 ) -> Any:
     """
     Call a single instance of a parallel task
@@ -274,9 +274,9 @@ def call_single_parallel_task(
             The task to execute.
         task_pars:
             The parameters to pass on to the task.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors).
 
@@ -290,17 +290,18 @@ def call_single_parallel_task(
                             information to the TaskExecutionError, such as task
                             order and name.
         JobExecutionError: If the wrapped task raises a job-related error.
-        RuntimeError: If the `workflow_dir` is falsy.
+        RuntimeError: If the `workflow_dir_local` is falsy.
     """
-    if not workflow_dir:
+    if not workflow_dir_local:
         raise RuntimeError
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
         component=component,
     )
 
@@ -363,8 +364,8 @@ def call_parallel_task(
     executor: Executor,
     wftask: WorkflowTask,
     task_pars_depend: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
     logger_name: Optional[str] = None,
 ) -> TaskParameters:
@@ -387,9 +388,9 @@ def call_parallel_task(
             The parallel task to run.
         task_pars_depend:
             The task parameters to be passed on to the parallel task.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors).
         submit_setup_call:
@@ -405,8 +406,8 @@ def call_parallel_task(
     """
     logger = get_logger(logger_name)
 
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     try:
         component_list = task_pars_depend.metadata[
@@ -424,8 +425,8 @@ def call_parallel_task(
     try:
         extra_setup = submit_setup_call(
             wftask=wftask,
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
         )
     except Exception as e:
         tb = "".join(traceback.format_tb(e.__traceback__))
@@ -443,8 +444,8 @@ def call_parallel_task(
         call_single_parallel_task,
         wftask=wftask,
         task_pars=actual_task_pars_depend,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
     )
 
     # Submit tasks for execution. Note that `for _ in map_iter:
@@ -511,16 +512,17 @@ def execute_tasks(
     executor: Executor,
     task_list: list[WorkflowTask],
     task_pars: TaskParameters,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
     logger_name: str,
 ) -> TaskParameters:
     """
     Submit a list of WorkflowTasks for execution
 
-    **Note:** At the end of each task, write current metadata to `working_dir /
-    METADATA_FILENAME`, so that they can be read as part of the [`get_job`
+    **Note:** At the end of each task, write current metadata to
+    `workflow_dir_local / METADATA_FILENAME`, so that they can be read as part
+    of the [`get_job`
     endpoint](../../api/v1/job/#fractal_server.app.routes.api.v1.job.get_job).
 
     Arguments:
@@ -531,12 +533,12 @@ def execute_tasks(
             The list of wftasks to be run
         task_pars:
             The task parameters to be passed on to the first task of the list.
-        workflow_dir:
+        workflow_dir_local:
             The server-side working directory for workflow execution.
-        workflow_dir_user:
+        workflow_dir_remote:
             The user-side working directory for workflow execution (only
             relevant for multi-user executors). If `None`, it is set to be
-            equal to `workflow_dir`.
+            equal to `workflow_dir_local`.
         submit_setup_call:
             An optional function that computes configuration parameters for
             the executor.
@@ -548,8 +550,8 @@ def execute_tasks(
             A TaskParameters object which constitutes the output of the last
             task in the list.
     """
-    if not workflow_dir_user:
-        workflow_dir_user = workflow_dir
+    if not workflow_dir_remote:
+        workflow_dir_remote = workflow_dir_local
 
     logger = get_logger(logger_name)
 
@@ -565,8 +567,8 @@ def execute_tasks(
                 executor=executor,
                 wftask=this_wftask,
                 task_pars_depend=current_task_pars,
-                workflow_dir=workflow_dir,
-                workflow_dir_user=workflow_dir_user,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
                 submit_setup_call=submit_setup_call,
                 logger_name=logger_name,
             )
@@ -575,8 +577,8 @@ def execute_tasks(
             try:
                 extra_setup = submit_setup_call(
                     wftask=this_wftask,
-                    workflow_dir=workflow_dir,
-                    workflow_dir_user=workflow_dir_user,
+                    workflow_dir_local=workflow_dir_local,
+                    workflow_dir_remote=workflow_dir_remote,
                 )
             except Exception as e:
                 tb = "".join(traceback.format_tb(e.__traceback__))
@@ -594,8 +596,8 @@ def execute_tasks(
                 call_single_task,
                 wftask=this_wftask,
                 task_pars=current_task_pars,
-                workflow_dir=workflow_dir,
-                workflow_dir_user=workflow_dir_user,
+                workflow_dir_local=workflow_dir_local,
+                workflow_dir_remote=workflow_dir_remote,
                 logger_name=logger_name,
                 **extra_setup,
             )
@@ -607,11 +609,11 @@ def execute_tasks(
         )
 
         # Write most recent metadata to METADATA_FILENAME
-        with open(workflow_dir / METADATA_FILENAME, "w") as f:
+        with open(workflow_dir_local / METADATA_FILENAME, "w") as f:
             json.dump(current_task_pars.metadata, f, indent=2)
 
         # Write most recent metadata to HISTORY_FILENAME
-        with open(workflow_dir / HISTORY_FILENAME, "w") as f:
+        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
             json.dump(current_task_pars.history, f, indent=2)
 
     return current_task_pars

fractal_server/app/runner/v1/_local/__init__.py

@@ -40,7 +40,7 @@ def _process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
+    workflow_dir_local: Path,
     first_task_index: int,
     last_task_index: int,
 ) -> dict[str, Any]:
@@ -66,8 +66,8 @@ def _process_workflow(
                 metadata=input_metadata,
                 history=input_history,
             ),
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
             logger_name=logger_name,
             submit_setup_call=_local_submit_setup,
         )
@@ -85,8 +85,8 @@ async def process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
     user_cache_dir: Optional[str] = None,
@@ -115,12 +115,13 @@ async def process_workflow(
             Initial metadata, passed to the first task
         logger_name:
             Name of the logger to log information on the run to
-        workflow_dir:
+        workflow_dir_local:
             Working directory for this run.
-        workflow_dir_user:
+        workflow_dir_remote:
             Working directory for this run, on the user side. This argument is
             present for compatibility with the standard backend interface, but
-            for the `local` backend it cannot be different from `workflow_dir`.
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
         slurm_user:
             Username to impersonate to run the workflow. This argument is
             present for compatibility with the standard backend interface, but
@@ -157,10 +158,10 @@ async def process_workflow(
             of the workflow
     """
 
-    if workflow_dir_user and (workflow_dir_user != workflow_dir):
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
         raise NotImplementedError(
             "Local backend does not support different directories "
-            f"{workflow_dir=} and {workflow_dir_user=}"
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
         )
 
     # Set values of first_task_index and last_task_index
@@ -178,7 +179,7 @@ async def process_workflow(
         input_metadata=input_metadata,
         input_history=input_history,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
+        workflow_dir_local=workflow_dir_local,
         first_task_index=first_task_index,
         last_task_index=last_task_index,
     )

fractal_server/app/runner/v1/_local/_submit_setup.py

@@ -21,8 +21,8 @@ from ._local_config import get_local_backend_config
 def _local_submit_setup(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Optional[Path] = None,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Optional[Path] = None,
+    workflow_dir_remote: Optional[Path] = None,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -31,9 +31,9 @@ def _local_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Not used in this function.
-        workflow_dir_user:
+        workflow_dir_remote:
             Not used in this function.
 
     Returns:

fractal_server/app/runner/v1/_slurm/__init__.py

@@ -50,8 +50,8 @@ def _process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     first_task_index: int,
     last_task_index: int,
     slurm_user: Optional[str] = None,
@@ -86,8 +86,8 @@ def _process_workflow(
         keep_logs=True,
         slurm_user=slurm_user,
         user_cache_dir=user_cache_dir,
-        working_dir=workflow_dir,
-        working_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
@@ -102,8 +102,8 @@ def _process_workflow(
                 metadata=input_metadata,
                 history=input_history,
             ),
-            workflow_dir=workflow_dir,
-            workflow_dir_user=workflow_dir_user,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_remote,
             submit_setup_call=_slurm_submit_setup,
             logger_name=logger_name,
         )
@@ -121,8 +121,8 @@ async def process_workflow(
     input_metadata: dict[str, Any],
     input_history: list[dict[str, Any]],
     logger_name: str,
-    workflow_dir: Path,
-    workflow_dir_user: Optional[Path] = None,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
     slurm_account: Optional[str] = None,
@@ -152,8 +152,8 @@ async def process_workflow(
         input_metadata=input_metadata,
         input_history=input_history,
         logger_name=logger_name,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         slurm_user=slurm_user,
         slurm_account=slurm_account,
         user_cache_dir=user_cache_dir,
@@ -166,8 +166,8 @@ async def process_workflow(
 
 def get_slurm_config(
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     config_path: Optional[Path] = None,
 ) -> SlurmConfig:
     """
@@ -187,13 +187,13 @@ def get_slurm_config(
         wftask:
             WorkflowTask for which the SLURM configuration is is to be
             prepared.
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
         config_path:
             Path of aFractal  SLURM configuration file; if `None`, use
             `FRACTAL_SLURM_CONFIG_FILE` variable from settings.

fractal_server/app/runner/v1/_slurm/_submit_setup.py

@@ -24,8 +24,8 @@ from fractal_server.app.models.v1 import WorkflowTask
 def _slurm_submit_setup(
     *,
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
 ) -> dict[str, object]:
     """
     Collect WorfklowTask-specific configuration parameters from different
@@ -43,13 +43,13 @@ def _slurm_submit_setup(
     Arguments:
         wftask:
             WorkflowTask for which the configuration is to be assembled
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
 
     Returns:
         submit_setup_dict:
@@ -61,15 +61,16 @@ def _slurm_submit_setup(
     # Get SlurmConfig object
     slurm_config = get_slurm_config(
         wftask=wftask,
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
     )
 
     # Get TaskFiles object
     task_files = get_task_file_paths(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
         task_order=wftask.order,
+        task_name=wftask.task.name,
     )
 
     # Prepare and return output dictionary

fractal_server/app/runner/v1/_slurm/get_slurm_config.py

@@ -17,8 +17,8 @@ from fractal_server.app.runner.executors.slurm._slurm_config import (
 
 def get_slurm_config(
     wftask: WorkflowTask,
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
     config_path: Optional[Path] = None,
 ) -> SlurmConfig:
     """
@@ -38,13 +38,13 @@ def get_slurm_config(
         wftask:
             WorkflowTask for which the SLURM configuration is is to be
             prepared.
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
             files (inputs, outputs, errors, and all meta files related to the
             job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
         config_path:
             Path of aFractal  SLURM configuration file; if `None`, use
             `FRACTAL_SLURM_CONFIG_FILE` variable from settings.