fractal-server 2.13.0__py3-none-any.whl → 2.14.0a0__py3-none-any.whl

fractal_server/app/runner/task_files.py

@@ -2,6 +2,8 @@ from pathlib import Path
 from typing import Optional
 from typing import Union
 
+from pydantic import BaseModel
+
 from fractal_server.string_tools import sanitize_string
 
 
@@ -17,108 +19,80 @@ def task_subfolder_name(order: Union[int, str], task_name: str) -> str:
     return f"{order}_{task_name_slug}"
 
 
-class TaskFiles:
+class TaskFiles(BaseModel):
     """
-    Group all file paths pertaining to a task
-
-    Attributes:
-        workflow_dir_local:
-            Server-owned directory to store all task-execution-related relevant
-            files. Note: users cannot write directly to this folder.
-        workflow_dir_remote:
-            User-side directory with the same scope as `workflow_dir_local`,
-            and where a user can write.
-        subfolder_name:
-            Name of task-specific subfolder
-        remote_subfolder:
-            Path to user-side task-specific subfolder
-        task_name:
-            Name of the task
-        task_order:
-            Positional order of the task within a workflow.
-        component:
-            Specific component to run the task for (relevant for tasks to be
-            executed in parallel over many components).
-        file_prefix:
-            Prefix for all task-related files.
-        args:
-            Path for input json file.
-        metadiff:
-            Path for output json file with metadata update.
-        out:
-            Path for task-execution stdout.
-        err:
-            Path for task-execution stderr.
+    Group all file paths pertaining to a task     FIXME
     """
 
-    workflow_dir_local: Path
-    workflow_dir_remote: Path
-    remote_subfolder: Path
-    subfolder_name: str
+    # Parent directory
+    root_dir_local: Path
+    root_dir_remote: Path
+
+    # Per-wftask
     task_name: str
-    task_order: Optional[int] = None
+    task_order: int
+
+    # Per-single-component
     component: Optional[str] = None
 
-    file_prefix: str
-    file_prefix_with_subfolder: str
-    args: Path
-    out: Path
-    err: Path
-    log: Path
-    metadiff: Path
-
-    def __init__(
-        self,
-        workflow_dir_local: Path,
-        workflow_dir_remote: Path,
-        task_name: str,
-        task_order: Optional[int] = None,
-        component: Optional[str] = None,
-    ):
-        self.workflow_dir_local = workflow_dir_local
-        self.workflow_dir_remote = workflow_dir_remote
-        self.task_order = task_order
-        self.task_name = task_name
-        self.component = component
-
-        if self.component is not None:
-            component_safe = sanitize_string(str(self.component))
-            component_safe = f"_par_{component_safe}"
-        else:
-            component_safe = ""
-
-        if self.task_order is not None:
-            order = str(self.task_order)
-        else:
-            order = "0"
-        self.file_prefix = f"{order}{component_safe}"
-        self.subfolder_name = task_subfolder_name(
-            order=order, task_name=self.task_name
-        )
-        self.remote_subfolder = self.workflow_dir_remote / self.subfolder_name
-        self.args = self.remote_subfolder / f"{self.file_prefix}.args.json"
-        self.out = self.remote_subfolder / f"{self.file_prefix}.out"
-        self.err = self.remote_subfolder / f"{self.file_prefix}.err"
-        self.log = self.remote_subfolder / f"{self.file_prefix}.log"
-        self.metadiff = (
-            self.remote_subfolder / f"{self.file_prefix}.metadiff.json"
+    def _check_component(self):
+        if self.component is None:
+            raise ValueError("`component` cannot be None")
+
+    @property
+    def subfolder_name(self) -> str:
+        order = str(self.task_order or 0)
+        return task_subfolder_name(
+            order=order,
+            task_name=self.task_name,
         )
 
+    @property
+    def wftask_subfolder_remote(self) -> Path:
+        return self.root_dir_remote / self.subfolder_name
 
-def get_task_file_paths(
-    workflow_dir_local: Path,
-    workflow_dir_remote: Path,
-    task_name: str,
-    task_order: Optional[int] = None,
-    component: Optional[str] = None,
-) -> TaskFiles:
-    """
-    Return the corrisponding TaskFiles object
-    """
-    return TaskFiles(
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
-        task_name=task_name,
-        task_order=task_order,
-        component=component,
-    )
+    @property
+    def wftask_subfolder_local(self) -> Path:
+        return self.root_dir_local / self.subfolder_name
+
+    @property
+    def log_file_local(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_local / f"{self.component}-log.txt"
+        ).as_posix()
+
+    @property
+    def log_file_remote(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_remote / f"{self.component}-log.txt"
+        ).as_posix()
+
+    @property
+    def args_file_local(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_local / f"{self.component}-args.json"
+        ).as_posix()
+
+    @property
+    def args_file_remote(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_remote / f"{self.component}-args.json"
+        ).as_posix()
+
+    @property
+    def metadiff_file_local(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_local / f"{self.component}-metadiff.json"
+        ).as_posix()
+
+    @property
+    def metadiff_file_remote(self) -> str:
+        self._check_component()
+        return (
+            self.wftask_subfolder_remote / f"{self.component}-metadiff.json"
+        ).as_posix()

fractal_server/app/runner/v2/__init__.py

@@ -27,13 +27,12 @@ from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from ..executors.slurm.sudo._subprocess_run_as_user import _mkdir_as_user
+from ..executors.slurm_sudo._subprocess_run_as_user import _mkdir_as_user
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
 from ._slurm_sudo import process_workflow as slurm_sudo_process_workflow
-from .handle_failed_job import mark_last_wftask_as_failed
 from fractal_server import __VERSION__
 from fractal_server.app.models import UserSettings
 
@@ -70,6 +69,7 @@ def submit_workflow(
     workflow_id: int,
     dataset_id: int,
     job_id: int,
+    user_id: int,
     user_settings: UserSettings,
     worker_init: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -90,6 +90,8 @@ def submit_workflow(
         job_id:
             Id of the job record which stores the state for the current
             workflow application.
+        user_id:
+            User ID.
         worker_init:
             Custom executor parameters that get parsed before the execution of
             each task.
@@ -198,7 +200,7 @@ def submit_workflow(
                     f"{settings.FRACTAL_RUNNER_BACKEND}."
                 )
 
-            # Create all tasks subfolders
+            # Create all tasks subfolders # FIXME: do this with Runner
             for order in range(job.first_task_index, job.last_task_index + 1):
                 this_wftask = workflow.task_list[order]
                 task_name = this_wftask.task.name
@@ -216,10 +218,7 @@ def submit_workflow(
                         folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
                         user=slurm_user,
                     )
-                else:
-                    # Create local subfolder (with standard permission set)
-                    (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir()
-                    logger.info("Skip remote-subfolder creation")
+
         except Exception as e:
             error_type = type(e).__name__
             fail_job(
@@ -312,6 +311,7 @@ def submit_workflow(
         process_workflow(
             workflow=workflow,
             dataset=dataset,
+            user_id=user_id,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
             workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
@@ -341,10 +341,6 @@ def submit_workflow(
         logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
 
-        mark_last_wftask_as_failed(
-            dataset_id=dataset_id,
-            logger_name=logger_name,
-        )
         exception_args_string = "\n".join(e.args)
         log_msg = (
             f"TASK ERROR: "
@@ -357,10 +353,7 @@ def submit_workflow(
     except JobExecutionError as e:
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')
-        mark_last_wftask_as_failed(
-            dataset_id=dataset_id,
-            logger_name=logger_name,
-        )
+
         fail_job(
             db=db_sync,
             job=job,
@@ -374,10 +367,7 @@ def submit_workflow(
     except Exception:
         logger.debug(f'FAILED workflow "{workflow.name}", unknown error.')
         logger.info(f'Workflow "{workflow.name}" failed (unkwnon error).')
-        mark_last_wftask_as_failed(
-            dataset_id=dataset_id,
-            logger_name=logger_name,
-        )
+
         current_traceback = traceback.format_exc()
         fail_job(
             db=db_sync,

fractal_server/app/runner/v2/_local.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+from typing import Optional
+
+from ...models.v2 import DatasetV2
+from ...models.v2 import WorkflowV2
+from ..executors.local._submit_setup import _local_submit_setup
+from ..executors.local.runner import LocalRunner
+from ..set_start_and_last_task_index import set_start_and_last_task_index
+from .runner import execute_tasks_v2
+from fractal_server.images.models import AttributeFiltersType
+
+
+def process_workflow(
+    *,
+    workflow: WorkflowV2,
+    dataset: DatasetV2,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Optional[Path] = None,
+    first_task_index: Optional[int] = None,
+    last_task_index: Optional[int] = None,
+    logger_name: str,
+    job_attribute_filters: AttributeFiltersType,
+    user_id: int,
+    **kwargs,
+) -> None:
+    """
+    Run a workflow through
+
+    Args:
+        workflow:
+            The workflow to be run
+        dataset:
+            Initial dataset.
+        workflow_dir_local:
+            Working directory for this run.
+        workflow_dir_remote:
+            Working directory for this run, on the user side. This argument is
+            present for compatibility with the standard backend interface, but
+            for the `local` backend it cannot be different from
+            `workflow_dir_local`.
+        first_task_index:
+            Positional index of the first task to execute; if `None`, start
+            from `0`.
+        last_task_index:
+            Positional index of the last task to execute; if `None`, proceed
+            until the last task.
+        logger_name: Logger name
+        user_id:
+
+    Raises:
+        TaskExecutionError: wrapper for errors raised during tasks' execution
+                            (positive exit codes).
+        JobExecutionError: wrapper for errors raised by the tasks' executors
+                           (negative exit codes).
+    """
+
+    if workflow_dir_remote and (workflow_dir_remote != workflow_dir_local):
+        raise NotImplementedError(
+            "Local backend does not support different directories "
+            f"{workflow_dir_local=} and {workflow_dir_remote=}"
+        )
+
+    # Set values of first_task_index and last_task_index
+    num_tasks = len(workflow.task_list)
+    first_task_index, last_task_index = set_start_and_last_task_index(
+        num_tasks,
+        first_task_index=first_task_index,
+        last_task_index=last_task_index,
+    )
+
+    with LocalRunner(root_dir_local=workflow_dir_local) as runner:
+        execute_tasks_v2(
+            wf_task_list=workflow.task_list[
+                first_task_index : (last_task_index + 1)
+            ],
+            dataset=dataset,
+            runner=runner,
+            workflow_dir_local=workflow_dir_local,
+            workflow_dir_remote=workflow_dir_local,
+            logger_name=logger_name,
+            submit_setup_call=_local_submit_setup,
+            job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
+        )

fractal_server/app/runner/v2/{_slurm_ssh/__init__.py → _slurm_ssh.py}

@@ -19,14 +19,14 @@ Executor objects.
 from pathlib import Path
 from typing import Optional
 
-from .....ssh._fabric import FractalSSH
-from ....models.v2 import DatasetV2
-from ....models.v2 import WorkflowV2
-from ...exceptions import JobExecutionError
-from ...executors.slurm.ssh.executor import FractalSlurmSSHExecutor
-from ...set_start_and_last_task_index import set_start_and_last_task_index
-from ..runner import execute_tasks_v2
-from ._submit_setup import _slurm_submit_setup
+from ....ssh._fabric import FractalSSH
+from ...models.v2 import DatasetV2
+from ...models.v2 import WorkflowV2
+from ..exceptions import JobExecutionError
+from ..executors.slurm_common._submit_setup import _slurm_submit_setup
+from ..executors.slurm_ssh.executor import FractalSlurmSSHExecutor
+from ..set_start_and_last_task_index import set_start_and_last_task_index
+from .runner import execute_tasks_v2
 from fractal_server.images.models import AttributeFiltersType
 from fractal_server.logger import set_logger
 
@@ -45,10 +45,8 @@ def process_workflow(
     job_attribute_filters: AttributeFiltersType,
     fractal_ssh: FractalSSH,
     worker_init: Optional[str] = None,
-    # Not used
-    user_cache_dir: Optional[str] = None,
-    slurm_user: Optional[str] = None,
-    slurm_account: Optional[str] = None,
+    user_id: int,
+    **kwargs,  # not used
 ) -> None:
     """
     Process workflow (SLURM backend public interface)
@@ -88,10 +86,11 @@ def process_workflow(
                 first_task_index : (last_task_index + 1)
             ],
             dataset=dataset,
-            executor=executor,
+            runner=executor,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
         )

fractal_server/app/runner/v2/{_slurm_sudo/__init__.py → _slurm_sudo.py}

@@ -19,12 +19,12 @@ Executor objects.
 from pathlib import Path
 from typing import Optional
 
-from ....models.v2 import DatasetV2
-from ....models.v2 import WorkflowV2
-from ...executors.slurm.sudo.executor import FractalSlurmSudoExecutor
-from ...set_start_and_last_task_index import set_start_and_last_task_index
-from ..runner import execute_tasks_v2
-from ._submit_setup import _slurm_submit_setup
+from ...models.v2 import DatasetV2
+from ...models.v2 import WorkflowV2
+from ..executors.slurm_common._submit_setup import _slurm_submit_setup
+from ..executors.slurm_sudo.runner import RunnerSlurmSudo
+from ..set_start_and_last_task_index import set_start_and_last_task_index
+from .runner import execute_tasks_v2
 from fractal_server.images.models import AttributeFiltersType
 
 
@@ -38,6 +38,7 @@ def process_workflow(
     last_task_index: Optional[int] = None,
     logger_name: str,
     job_attribute_filters: AttributeFiltersType,
+    user_id: int,
     # Slurm-specific
     user_cache_dir: Optional[str] = None,
     slurm_user: Optional[str] = None,
@@ -64,13 +65,11 @@ def process_workflow(
     if isinstance(worker_init, str):
         worker_init = worker_init.split("\n")
 
-    with FractalSlurmSudoExecutor(
-        debug=True,
-        keep_logs=True,
+    with RunnerSlurmSudo(
         slurm_user=slurm_user,
         user_cache_dir=user_cache_dir,
-        workflow_dir_local=workflow_dir_local,
-        workflow_dir_remote=workflow_dir_remote,
+        root_dir_local=workflow_dir_local,
+        root_dir_remote=workflow_dir_remote,
         common_script_lines=worker_init,
         slurm_account=slurm_account,
     ) as executor:
@@ -79,10 +78,11 @@ def process_workflow(
                 first_task_index : (last_task_index + 1)
             ],
             dataset=dataset,
-            executor=executor,
+            runner=executor,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,
             logger_name=logger_name,
             submit_setup_call=_slurm_submit_setup,
             job_attribute_filters=job_attribute_filters,
+            user_id=user_id,
         )