fractal-server 2.14.0a15__tar.gz → 2.14.0a17__tar.gz

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a15
+Version: 2.14.0a17
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a17/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a17"

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/fractal_server/app/runner/executors/local/runner.py

@@ -11,6 +11,9 @@ from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -69,7 +72,12 @@ class LocalRunner(BaseRunner):
         workdir_local = task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
+        # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # SUBMISSION PHASE
         future = self.executor.submit(
@@ -155,17 +163,35 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                current_task_files = list_task_files[positional_index]
-                current_task_files.prefix = (
-                    f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
-                )
+                list_task_files[
+                    positional_index
+                ].prefix = f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
                 future = self.executor.submit(
                     func,
                     parameters=kwargs,
-                    remote_files=current_task_files.remote_files_dict,
+                    remote_files=list_task_files[
+                        positional_index
+                    ].remote_files_dict,
                 )
                 active_futures[positional_index] = future
 
+                if task_type == "parallel":
+                    # FIXME: replace loop with a `bulk_update_history_unit`
+                    # function
+                    update_logfile_of_history_unit(
+                        history_unit_id=history_unit_ids[positional_index],
+                        logfile=list_task_files[
+                            positional_index
+                        ].log_file_local,
+                    )
+                else:
+                    logger.debug(
+                        f"Unclear what logfile to associate to {task_type=} "
+                        "within multisubmit (see issue #2382)."
+                    )
+                    # FIXME: Improve definition for compound tasks
+                    pass
+
             while active_futures:
                 # FIXME: add shutdown detection
                 # if file exists: cancel all futures, and raise

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,12 +23,18 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 
+SHUTDOWN_ERROR_MESSAGE = "Failed due to job-execution shutdown."
+SHUTDOWN_EXCEPTION = JobExecutionError(SHUTDOWN_ERROR_MESSAGE)
+
 logger = set_logger(__name__)
 
 # FIXME: Transform several logger.info into logger.debug.
@@ -91,6 +97,7 @@ class BaseSlurmRunner(BaseRunner):
 
     def run_squeue(self, job_ids: list[str]) -> tuple[bool, str]:
         # FIXME: review different cases (exception vs no job found)
+        # FIXME: Fail for empty list
         job_id_single_str = ",".join([str(j) for j in job_ids])
         cmd = (
             f"squeue --noheader --format='%i %T' --jobs {job_id_single_str}"
@@ -225,9 +232,11 @@ class BaseSlurmRunner(BaseRunner):
         logger.info(script_lines)
 
         # Always print output of `uname -n` and `pwd`
-        script_lines.append("Hostname: $(uname -n)\n")
-        script_lines.append("Current directory : $(pwd)\n")
-        script_lines.append('Start time: $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
+        script_lines.append('\necho "Hostname: $(uname -n)"')
+        script_lines.append('echo "Current directory : $(pwd)"')
+        script_lines.append(
+            'echo "Start time: $(date +"%Y-%m-%dT%H:%M:%S%z")"'
+        )
 
         # Complete script preamble
         script_lines.append("\n")
@@ -242,7 +251,9 @@ class BaseSlurmRunner(BaseRunner):
             )
         script_lines.append("wait\n")
         script = "\n".join(script_lines)
-        script_lines.append('End time:   $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
+        script_lines.append(
+            'echo "End time:   $(date +"%Y-%m-%dT%H:%M:%S%z")"'
+        )
 
         # Write submission script
         with open(slurm_job.slurm_submission_script_local, "w") as f:
@@ -332,7 +343,10 @@ class BaseSlurmRunner(BaseRunner):
             pass
 
     def _postprocess_single_task(
-        self, *, task: SlurmTask
+        self,
+        *,
+        task: SlurmTask,
+        was_job_scancelled: bool = False,
     ) -> tuple[Any, Exception]:
         try:
             with open(task.output_pickle_file_local, "rb") as f:
@@ -344,17 +358,24 @@ class BaseSlurmRunner(BaseRunner):
             else:
                 exception = _handle_exception_proxy(output)
                 return None, exception
+
         except Exception as e:
             exception = JobExecutionError(f"ERROR, {str(e)}")
+            # If job was scancelled and task failed, replace
+            # exception with a shutdown-related one.
+            if was_job_scancelled:
+                logger.debug(
+                    "Replacing exception with a shutdown-related one, "
+                    f"for {task.index=}."
+                )
+                exception = SHUTDOWN_EXCEPTION
+
             return None, exception
         finally:
-            pass
-            # FIXME: Re-include unlinks of pickle files
-            # Path(task.input_pickle_file_local).unlink(missing_ok=True)
-            # Path(task.output_pickle_file_local).unlink(missing_ok=True)
+            Path(task.input_pickle_file_local).unlink(missing_ok=True)
+            Path(task.output_pickle_file_local).unlink(missing_ok=True)
 
     def is_shutdown(self) -> bool:
-        # FIXME: shutdown is not implemented
         return self.shutdown_file.exists()
 
     @property
@@ -385,7 +406,14 @@ class BaseSlurmRunner(BaseRunner):
             raise JobExecutionError("Unexpected branch: jobs should be empty.")
 
         if self.is_shutdown():
-            raise JobExecutionError("Cannot continue after shutdown.")
+            with next(get_sync_db()) as db:
+                update_status_of_history_unit(
+                    history_unit_id=history_unit_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
+
+            return None, SHUTDOWN_EXCEPTION
 
         # Validation phase
         self.validate_submit_parameters(
@@ -401,6 +429,10 @@ class BaseSlurmRunner(BaseRunner):
 
         # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # Submission phase
         slurm_job = SlurmJob(
@@ -437,19 +469,29 @@ class BaseSlurmRunner(BaseRunner):
         # Retrieval phase
         logger.info("[submit] START retrieval phase")
         while len(self.jobs) > 0:
+
+            # Handle shutdown
+            scancelled_job_ids = []
             if self.is_shutdown():
-                self.scancel_jobs()
+                logger.info("[submit] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[submit] {scancelled_job_ids=}")
+
+            # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
-            logger.info(f"{finished_job_ids=}")
+            logger.debug(f"[submit] {finished_job_ids=}")
+
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
-                    logger.info(f"Now process {slurm_job_id=}")
+                    logger.debug(f"[submit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
-
                     self._copy_files_from_remote_to_local(slurm_job)
+                    was_job_scancelled = slurm_job_id in scancelled_job_ids
                     result, exception = self._postprocess_single_task(
-                        task=slurm_job.tasks[0]
+                        task=slurm_job.tasks[0],
+                        was_job_scancelled=was_job_scancelled,
                     )
+
                     if exception is not None:
                         update_status_of_history_unit(
                             history_unit_id=history_unit_id,
@@ -477,13 +519,29 @@ class BaseSlurmRunner(BaseRunner):
         list_task_files: list[TaskFiles],
         task_type: Literal["parallel", "compound", "converter_compound"],
         config: SlurmConfig,
-    ):
+    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
 
         if len(self.jobs) > 0:
             raise RuntimeError(
-                f"Cannot run .multisubmit when {len(self.jobs)=}"
+                f"Cannot run `multisubmit` when {len(self.jobs)=}"
             )
 
+        if self.is_shutdown():
+            if task_type == "parallel":
+                with next(get_sync_db()) as db:
+                    # FIXME: Replace with bulk function
+                    for history_unit_id in history_unit_ids:
+                        update_status_of_history_unit(
+                            history_unit_id=history_unit_id,
+                            status=HistoryUnitStatus.FAILED,
+                            db_sync=db,
+                        )
+            results = {}
+            exceptions = {
+                ind: SHUTDOWN_EXCEPTION for ind in range(len(list_parameters))
+            }
+            return results, exceptions
+
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
             task_type=task_type,
@@ -550,18 +608,17 @@ class BaseSlurmRunner(BaseRunner):
             tasks = []
             for ind_chunk, parameters in enumerate(chunk):
                 index = (ind_batch * batch_size) + ind_chunk
-                current_task_files = list_task_files[index]
-                current_task_files.prefix = prefix
+                list_task_files[index].prefix = prefix
                 tasks.append(
                     SlurmTask(
                         prefix=prefix,
                         index=index,
-                        component=current_task_files.component,
+                        component=list_task_files[index].component,
                         workdir_local=workdir_local,
                         workdir_remote=workdir_remote,
                         parameters=parameters,
                         zarr_url=parameters["zarr_url"],
-                        task_files=current_task_files,
+                        task_files=list_task_files[index],
                     ),
                 )
 
@@ -576,6 +633,21 @@ class BaseSlurmRunner(BaseRunner):
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
+        if task_type == "parallel":
+            # FIXME: replace loop with a `bulk_update_history_unit` function
+            for ind, task_files in enumerate(list_task_files):
+                update_logfile_of_history_unit(
+                    history_unit_id=history_unit_ids[ind],
+                    logfile=task_files.log_file_local,
+                )
+        else:
+            logger.debug(
+                f"Unclear what logfile to associate to {task_type=} "
+                "within multisubmit (see issue #2382)."
+            )
+            # FIXME: Improve definition for compound tasks
+            pass
+
         logger.info(f"END submission phase, {self.job_ids=}")
 
         # FIXME: replace this sleep a more precise check
@@ -584,22 +656,34 @@ class BaseSlurmRunner(BaseRunner):
         logger.warning(f"[submit] Now sleep {sleep_time} (FIXME)")
         time.sleep(sleep_time)
 
+        # FIXME: Could we merge the submit/multisubmit retrieval phases?
+
         # Retrieval phase
-        logger.info("START retrieval phase")
+        logger.info("[multisubmit] START retrieval phase")
         while len(self.jobs) > 0:
+
+            # Handle shutdown
+            scancelled_job_ids = []
             if self.is_shutdown():
-                self.scancel_jobs()
+                logger.info("[multisubmit] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[multisubmit] {scancelled_job_ids=}")
+
+            # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
-            logger.info(f"{finished_job_ids=}")
+            logger.debug(f"[multisubmit] {finished_job_ids=}")
+
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
-                    logger.info(f"Now processing {slurm_job_id=}")
+                    logger.info(f"[multisubmit] Now process {slurm_job_id=}")
                     slurm_job = self.jobs.pop(slurm_job_id)
                     self._copy_files_from_remote_to_local(slurm_job)
                     for task in slurm_job.tasks:
-                        logger.info(f"Now processing {task.index=}")
+                        logger.info(f"[multisubmit] Now process {task.index=}")
+                        was_job_scancelled = slurm_job_id in scancelled_job_ids
                         result, exception = self._postprocess_single_task(
-                            task=task
+                            task=task,
+                            was_job_scancelled=was_job_scancelled,
                         )
 
                         # Note: the relevant done/failed check is based on
@@ -659,11 +743,12 @@ class BaseSlurmRunner(BaseRunner):
             logger.error(error_msg)
             raise RuntimeError(error_msg)
 
-    def scancel_jobs(self) -> None:
+    def scancel_jobs(self) -> list[str]:
         logger.info("[scancel_jobs] START")
 
         if self.jobs:
-            scancel_string = " ".join(self.job_ids)
+            scancelled_job_ids = self.job_ids
+            scancel_string = " ".join(scancelled_job_ids)
             scancel_cmd = f"scancel {scancel_string}"
             logger.warning(f"Now scancel-ing SLURM jobs {scancel_string}")
             try:
@@ -673,5 +758,5 @@ class BaseSlurmRunner(BaseRunner):
                     "[scancel_jobs] `scancel` command failed. "
                     f"Original error:\n{str(e)}"
                 )
-
         logger.info("[scancel_jobs] END")
+        return scancelled_job_ids

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -140,7 +140,7 @@ class SudoSlurmRunner(BaseSlurmRunner):
                     f"Original error: {str(e)}"
                 )
 
-    def _run_remote_cmd(self, cmd: str):
+    def _run_remote_cmd(self, cmd: str) -> str:
         res = _run_command_as_user(
             cmd=cmd,
             user=self.slurm_user,
@@ -149,6 +149,6 @@ class SudoSlurmRunner(BaseSlurmRunner):
         )
         return res.stdout
 
-    def _run_local_cmd(self, cmd: str):
+    def _run_local_cmd(self, cmd: str) -> str:
         res = _subprocess_run_or_raise(cmd)
         return res.stdout

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/fractal_server/app/runner/v2/db_tools.py

@@ -3,11 +3,14 @@ from typing import Any
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 
+_CHUNK_SIZE = 2_000
+
 
 def update_status_of_history_run(
     *,
@@ -37,7 +40,18 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
-_CHUNK_SIZE = 2_000
+def update_logfile_of_history_unit(
+    *,
+    history_unit_id: int,
+    logfile: str,
+) -> None:
+    with next(get_sync_db()) as db_sync:
+        unit = db_sync.get(HistoryUnit, history_unit_id)
+        if unit is None:
+            raise ValueError(f"HistoryUnit {history_unit_id} not found.")
+        unit.logfile = logfile
+        db_sync.merge(unit)
+        db_sync.commit()
 
 
 def bulk_upsert_image_cache_fast(

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/fractal_server/app/runner/v2/runner_functions.py

@@ -172,7 +172,7 @@ def run_v2_task_non_parallel(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=task_files.log_file_local,
+            logfile=None,
             zarr_urls=zarr_urls,
         )
         db.add(history_unit)
@@ -275,7 +275,7 @@ def run_v2_task_parallel(
         HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=list_task_files[ind].log_file_local,
+            logfile=None,
             zarr_urls=[image["zarr_url"]],
         )
         for ind, image in enumerate(images)
@@ -401,8 +401,7 @@ def run_v2_task_compound(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            # FIXME: What about compute-task logs?
-            logfile=task_files_init.log_file_local,
+            logfile=None,
             zarr_urls=input_image_zarr_urls,
         )
         db.add(history_unit)

{fractal_server-2.14.0a15 → fractal_server-2.14.0a17}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a15"
+version = "2.14.0a17"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a15"
+current_version = "2.14.0a17"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a15/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a15"