fractal-server 2.0.6__py3-none-any.whl → 2.2.0__py3-none-any.whl

fractal_server/app/runner/task_files.py

@@ -1,32 +1,55 @@
 from pathlib import Path
 from typing import Optional
+from typing import Union
+
+from fractal_server.tasks.utils import slugify_task_name
 
 
 def sanitize_component(value: str) -> str:
     """
     Remove {" ", "/", "."} form a string, e.g. going from
     'plate.zarr/B/03/0' to 'plate_zarr_B_03_0'.
+
+    Args:
+        value: Input strig
     """
     return value.replace(" ", "_").replace("/", "_").replace(".", "_")
 
 
+def task_subfolder_name(order: Union[int, str], task_name: str) -> str:
+    """
+    Get name of task-specific subfolder.
+
+    Args:
+        order:
+        task_name:
+    """
+    task_name_slug = slugify_task_name(task_name)
+    return f"{order}_{task_name_slug}"
+
+
 class TaskFiles:
     """
     Group all file paths pertaining to a task
 
     Attributes:
-        workflow_dir:
+        workflow_dir_local:
             Server-owned directory to store all task-execution-related relevant
-            files (inputs, outputs, errors, and all meta files related to the
-            job execution). Note: users cannot write directly to this folder.
-        workflow_dir_user:
-            User-side directory with the same scope as `workflow_dir`, and
-            where a user can write.
+            files. Note: users cannot write directly to this folder.
+        workflow_dir_remote:
+            User-side directory with the same scope as `workflow_dir_local`,
+            and where a user can write.
+        subfolder_name:
+            Name of task-specific subfolder
+        remote_subfolder:
+            Path to user-side task-specific subfolder
+        task_name:
+            Name of the task
         task_order:
             Positional order of the task within a workflow.
         component:
-            Specific component to run the task for (relevant for tasks that
-            will be executed in parallel over many components).
+            Specific component to run the task for (relevant for tasks to be
+            executed in parallel over many components).
         file_prefix:
             Prefix for all task-related files.
         args:
@@ -39,12 +62,16 @@ class TaskFiles:
             Path for task-execution stderr.
     """
 
-    workflow_dir: Path
-    workflow_dir_user: Path
+    workflow_dir_local: Path
+    workflow_dir_remote: Path
+    remote_subfolder: Path
+    subfolder_name: str
+    task_name: str
     task_order: Optional[int] = None
     component: Optional[str] = None
 
     file_prefix: str
+    file_prefix_with_subfolder: str
     args: Path
     out: Path
     err: Path
@@ -53,14 +80,16 @@ class TaskFiles:
 
     def __init__(
         self,
-        workflow_dir: Path,
-        workflow_dir_user: Path,
+        workflow_dir_local: Path,
+        workflow_dir_remote: Path,
+        task_name: str,
         task_order: Optional[int] = None,
         component: Optional[str] = None,
     ):
-        self.workflow_dir = workflow_dir
-        self.workflow_dir_user = workflow_dir_user
+        self.workflow_dir_local = workflow_dir_local
+        self.workflow_dir_remote = workflow_dir_remote
         self.task_order = task_order
+        self.task_name = task_name
         self.component = component
 
         if self.component is not None:
@@ -72,32 +101,35 @@ class TaskFiles:
         if self.task_order is not None:
             order = str(self.task_order)
         else:
-            order = "task"
+            order = "0"
         self.file_prefix = f"{order}{component_safe}"
-        self.args = self.workflow_dir_user / f"{self.file_prefix}.args.json"
-        self.out = self.workflow_dir_user / f"{self.file_prefix}.out"
-        self.err = self.workflow_dir_user / f"{self.file_prefix}.err"
-        self.log = self.workflow_dir_user / f"{self.file_prefix}.log"
+        self.subfolder_name = task_subfolder_name(
+            order=order, task_name=self.task_name
+        )
+        self.remote_subfolder = self.workflow_dir_remote / self.subfolder_name
+        self.args = self.remote_subfolder / f"{self.file_prefix}.args.json"
+        self.out = self.remote_subfolder / f"{self.file_prefix}.out"
+        self.err = self.remote_subfolder / f"{self.file_prefix}.err"
+        self.log = self.remote_subfolder / f"{self.file_prefix}.log"
         self.metadiff = (
-            self.workflow_dir_user / f"{self.file_prefix}.metadiff.json"
+            self.remote_subfolder / f"{self.file_prefix}.metadiff.json"
         )
 
 
 def get_task_file_paths(
-    workflow_dir: Path,
-    workflow_dir_user: Path,
+    workflow_dir_local: Path,
+    workflow_dir_remote: Path,
+    task_name: str,
     task_order: Optional[int] = None,
     component: Optional[str] = None,
 ) -> TaskFiles:
     """
     Return the corrisponding TaskFiles object
-
-    This function is mainly used as a cache to avoid instantiating needless
-    objects.
     """
     return TaskFiles(
-        workflow_dir=workflow_dir,
-        workflow_dir_user=workflow_dir_user,
+        workflow_dir_local=workflow_dir_local,
+        workflow_dir_remote=workflow_dir_remote,
+        task_name=task_name,
         task_order=task_order,
         component=component,
     )

fractal_server/app/runner/v1/__init__.py

@@ -22,6 +22,10 @@ import traceback
 from pathlib import Path
 from typing import Optional
 
+from sqlalchemy.orm import Session as DBSyncSession
+
+from ....logger import get_logger
+from ....logger import reset_logger_handlers
 from ....logger import set_logger
 from ....syringe import Inject
 from ....utils import get_timestamp
@@ -33,7 +37,11 @@ from ...models.v1 import WorkflowTask
 from ...schemas.v1 import JobStatusTypeV1
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
+from ..executors.slurm._subprocess_run_as_user import (
+    _mkdir_as_user,
+)
 from ..filenames import WORKFLOW_LOG_FILENAME
+from ..task_files import task_subfolder_name
 from ._local import process_workflow as local_process_workflow
 from ._slurm import process_workflow as slurm_process_workflow
 from .common import close_job_logger
@@ -49,6 +57,27 @@ _backends["local"] = local_process_workflow
 _backends["slurm"] = slurm_process_workflow
 
 
+def fail_job(
+    *,
+    db: DBSyncSession,
+    job: ApplyWorkflow,
+    log_msg: str,
+    logger_name: str,
+    emit_log: bool = False,
+) -> None:
+    logger = get_logger(logger_name=logger_name)
+    if emit_log:
+        logger.error(log_msg)
+    reset_logger_handlers(logger)
+    job.status = JobStatusTypeV1.FAILED
+    job.end_timestamp = get_timestamp()
+    job.log = log_msg
+    db.merge(job)
+    db.commit()
+    db.close()
+    return
+
+
 async def submit_workflow(
     *,
     workflow_id: int,
@@ -87,21 +116,41 @@ async def submit_workflow(
             slurm backend.
     """
 
-    # Declare runner backend and set `process_workflow` function
-    settings = Inject(get_settings)
-    FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
-    if FRACTAL_RUNNER_BACKEND == "local":
-        process_workflow = local_process_workflow
-    elif FRACTAL_RUNNER_BACKEND == "slurm":
-        process_workflow = slurm_process_workflow
-    else:
-        raise RuntimeError(f"Invalid runner backend {FRACTAL_RUNNER_BACKEND=}")
+    logger_name = f"WF{workflow_id}_job{job_id}"
+    logger = set_logger(logger_name=logger_name)
 
     with next(DB.get_sync_db()) as db_sync:
 
         job: ApplyWorkflow = db_sync.get(ApplyWorkflow, job_id)
         if not job:
-            raise ValueError(f"Cannot fetch job {job_id} from database")
+            logger.error(f"ApplyWorkflow {job_id} does not exist")
+            return
+
+        settings = Inject(get_settings)
+        FRACTAL_RUNNER_BACKEND = settings.FRACTAL_RUNNER_BACKEND
+        if FRACTAL_RUNNER_BACKEND == "local":
+            process_workflow = local_process_workflow
+        elif FRACTAL_RUNNER_BACKEND == "slurm":
+            process_workflow = slurm_process_workflow
+        else:
+
+            if FRACTAL_RUNNER_BACKEND == "local_experimental":
+                log_msg = (
+                    f"{FRACTAL_RUNNER_BACKEND=} is not available for v1 jobs."
+                )
+            else:
+                log_msg = f"Invalid {FRACTAL_RUNNER_BACKEND=}"
+
+            fail_job(
+                job=job,
+                db=db_sync,
+                log_msg=log_msg,
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
+
+        # Declare runner backend and set `process_workflow` function
 
         input_dataset: Dataset = db_sync.get(Dataset, input_dataset_id)
         output_dataset: Dataset = db_sync.get(Dataset, output_dataset_id)
@@ -122,12 +171,9 @@ async def submit_workflow(
                 log_msg += (
                     f"Cannot fetch workflow {workflow_id} from database\n"
                 )
-            job.status = JobStatusTypeV1.FAILED
-            job.end_timestamp = get_timestamp()
-            job.log = log_msg
-            db_sync.merge(job)
-            db_sync.commit()
-            db_sync.close()
+            fail_job(
+                db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name
+            )
             return
 
         # Prepare some of process_workflow arguments
@@ -137,41 +183,53 @@ async def submit_workflow(
         # Define and create server-side working folder
         project_id = workflow.project_id
         timestamp_string = get_timestamp().strftime("%Y%m%d_%H%M%S")
-        WORKFLOW_DIR = (
-            settings.FRACTAL_RUNNER_WORKING_BASE_DIR
-            / (
-                f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
-                f"_{timestamp_string}"
-            )
-        ).resolve()
+        WORKFLOW_DIR_LOCAL = settings.FRACTAL_RUNNER_WORKING_BASE_DIR / (
+            f"proj_{project_id:07d}_wf_{workflow_id:07d}_job_{job_id:07d}"
+            f"_{timestamp_string}"
+        )
 
-        if WORKFLOW_DIR.exists():
-            raise RuntimeError(f"Workflow dir {WORKFLOW_DIR} already exists.")
+        if WORKFLOW_DIR_LOCAL.exists():
+            fail_job(
+                db=db_sync,
+                job=job,
+                log_msg=f"Workflow dir {WORKFLOW_DIR_LOCAL} already exists.",
+                logger_name=logger_name,
+                emit_log=True,
+            )
+            return
 
-        # Create WORKFLOW_DIR with 755 permissions
+        # Create WORKFLOW_DIR
         original_umask = os.umask(0)
-        WORKFLOW_DIR.mkdir(parents=True, mode=0o755)
+        WORKFLOW_DIR_LOCAL.mkdir(parents=True, mode=0o755)
         os.umask(original_umask)
 
-        # Define and create user-side working folder, if needed
+        # Define and create WORKFLOW_DIR_REMOTE
         if FRACTAL_RUNNER_BACKEND == "local":
-            WORKFLOW_DIR_USER = WORKFLOW_DIR
+            WORKFLOW_DIR_REMOTE = WORKFLOW_DIR_LOCAL
         elif FRACTAL_RUNNER_BACKEND == "slurm":
-
-            from ..executors.slurm._subprocess_run_as_user import (
-                _mkdir_as_user,
+            WORKFLOW_DIR_REMOTE = (
+                Path(user_cache_dir) / WORKFLOW_DIR_LOCAL.name
             )
+            _mkdir_as_user(folder=str(WORKFLOW_DIR_REMOTE), user=slurm_user)
 
-            WORKFLOW_DIR_USER = (
-                Path(user_cache_dir) / f"{WORKFLOW_DIR.name}"
-            ).resolve()
-            _mkdir_as_user(folder=str(WORKFLOW_DIR_USER), user=slurm_user)
-        else:
-            raise ValueError(f"{FRACTAL_RUNNER_BACKEND=} not supported")
+        # Create all tasks subfolders
+        for order in range(job.first_task_index, job.last_task_index + 1):
+            subfolder_name = task_subfolder_name(
+                order=order,
+                task_name=workflow.task_list[order].task.name,
+            )
+            original_umask = os.umask(0)
+            (WORKFLOW_DIR_LOCAL / subfolder_name).mkdir(mode=0o755)
+            os.umask(original_umask)
+            if FRACTAL_RUNNER_BACKEND == "slurm":
+                _mkdir_as_user(
+                    folder=str(WORKFLOW_DIR_REMOTE / subfolder_name),
+                    user=slurm_user,
+                )
 
         # Update db
-        job.working_dir = WORKFLOW_DIR.as_posix()
-        job.working_dir_user = WORKFLOW_DIR_USER.as_posix()
+        job.working_dir = WORKFLOW_DIR_LOCAL.as_posix()
+        job.working_dir_user = WORKFLOW_DIR_REMOTE.as_posix()
         db_sync.merge(job)
         db_sync.commit()
 
@@ -191,8 +249,7 @@ async def submit_workflow(
         db_sync.refresh(workflow)
 
         # Write logs
-        logger_name = f"WF{workflow_id}_job{job_id}"
-        log_file_path = WORKFLOW_DIR / WORKFLOW_LOG_FILENAME
+        log_file_path = WORKFLOW_DIR_LOCAL / WORKFLOW_LOG_FILENAME
         logger = set_logger(
             logger_name=logger_name,
             log_file_path=log_file_path,
@@ -239,8 +296,8 @@ async def submit_workflow(
             slurm_user=slurm_user,
             slurm_account=job.slurm_account,
             user_cache_dir=user_cache_dir,
-            workflow_dir=WORKFLOW_DIR,
-            workflow_dir_user=WORKFLOW_DIR_USER,
+            workflow_dir_local=WORKFLOW_DIR_LOCAL,
+            workflow_dir_remote=WORKFLOW_DIR_REMOTE,
             logger_name=logger_name,
             worker_init=worker_init,
             first_task_index=job.first_task_index,
@@ -291,19 +348,14 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-
         exception_args_string = "\n".join(e.args)
-        job.log = (
+        log_msg = (
             f"TASK ERROR: "
             f"Task name: {e.task_name}, "
             f"position in Workflow: {e.workflow_task_order}\n"
             f"TRACEBACK:\n{exception_args_string}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
 
     except JobExecutionError as e:
 
@@ -323,14 +375,13 @@ async def submit_workflow(
         )
 
         db_sync.merge(output_dataset)
-
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
         error = e.assemble_error()
-        job.log = f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}"
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(
+            db=db_sync,
+            job=job,
+            log_msg=f"JOB ERROR in Fractal job {job.id}:\nTRACEBACK:\n{error}",
+            logger_name=logger_name,
+        )
 
     except Exception:
 
@@ -353,14 +404,12 @@ async def submit_workflow(
 
         db_sync.merge(output_dataset)
 
-        job.status = JobStatusTypeV1.FAILED
-        job.end_timestamp = get_timestamp()
-        job.log = (
+        log_msg = (
             f"UNKNOWN ERROR in Fractal job {job.id}\n"
             f"TRACEBACK:\n{current_traceback}"
         )
-        db_sync.merge(job)
-        close_job_logger(logger)
-        db_sync.commit()
+        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
+
     finally:
         db_sync.close()
+        reset_logger_handlers(logger)