fractal-server 2.14.0a11__tar.gz → 2.14.0a13__tar.gz

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a11
+Version: 2.14.0a13
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a13/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a13"

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/fractal_server/app/runner/executors/base_runner.py

@@ -139,3 +139,25 @@ class BaseRunner(object):
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):
                 raise ValueError("Non-unique zarr_urls")
+
+    def validate_multisubmit_history_unit_ids(
+        self,
+        *,
+        history_unit_ids: list[int],
+        task_type: TaskTypeType,
+        list_parameters: list[dict[str, Any]],
+    ) -> None:
+        if task_type in ["compound", "converter_compound"]:
+            if len(history_unit_ids) != 1:
+                raise NotImplementedError(
+                    "We are breaking the assumption that compound/multisubmit "
+                    "is associated to a single HistoryUnit. This is not "
+                    "supported."
+                )
+            elif task_type == "parallel" and len(history_unit_ids) != len(
+                list_parameters
+            ):
+                raise ValueError(
+                    f"{len(history_unit_ids)=} differs from "
+                    f"{len(list_parameters)=}."
+                )

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/fractal_server/app/runner/executors/local/runner.py

@@ -115,23 +115,6 @@ class LocalRunner(BaseRunner):
         when we are in one of the "compound" cases
 
         """
-        # FIXME: De-duplicate this check
-        if task_type in ["compound", "converter_compound"]:
-            if len(history_unit_ids) != 1:
-                raise NotImplementedError(
-                    "We are breaking the assumption that compound/multisubmit "
-                    "is associated to a single HistoryUnit. This is not "
-                    "supported."
-                )
-        elif task_type == "parallel" and len(history_unit_ids) != len(
-            list_parameters
-        ):
-            raise ValueError(
-                f"{len(history_unit_ids)=} differs from "
-                f"{len(list_parameters)=}."
-            )
-
-        logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
@@ -139,6 +122,14 @@ class LocalRunner(BaseRunner):
             list_task_files=list_task_files,
         )
 
+        self.validate_multisubmit_history_unit_ids(
+            history_unit_ids=history_unit_ids,
+            task_type=task_type,
+            list_parameters=list_parameters,
+        )
+
+        logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
+
         workdir_local = list_task_files[0].wftask_subfolder_local
         if task_type == "parallel":
             workdir_local.mkdir()

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -17,7 +17,6 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 
 from ..slurm_common._check_jobs_status import get_finished_jobs
-from ..slurm_common._check_jobs_status import run_squeue
 from ._subprocess_run_as_user import _mkdir_as_user
 from ._subprocess_run_as_user import _run_command_as_user
 from fractal_server import __VERSION__
@@ -543,31 +542,21 @@ class RunnerSlurmSudo(BaseRunner):
                     task_files=task_files,
                 )
             ],
-        )  # TODO: replace with actual values (BASED ON TASKFILES)
-
+        )
         config.parallel_tasks_per_job = 1
         self._submit_single_sbatch(
             func,
             slurm_job=slurm_job,
             slurm_config=config,
         )
-        logger.debug("END SUBMISSION PHASE")
-        logger.debug(f"{self.jobs=}")
-        logger.debug(f"{self.job_ids=}")
-
-        # FIXME
-        jobs_that_started = set()
-        while len(jobs_that_started) != len(self.job_ids):
-            logger.debug("CALL SQUEUE")
-            res = run_squeue(self.job_ids)
-            new_jobs = set(out.split()[0] for out in res.stdout.splitlines())
-            jobs_that_started = jobs_that_started.union(new_jobs)
-            logger.debug(f"{new_jobs=}")
-            logger.debug(f"{len(jobs_that_started)=}")
-
-        logger.debug("START RETRIEVAL PHASE")
+        logger.info(f"END submission phase, {self.job_ids=}")
+
+        # FIXME: Replace with more robust/efficient logic
+        logger.warning("Now sleep 4 (FIXME)")
+        time.sleep(4)
 
         # Retrieval phase
+        logger.info("START retrieval phase")
         while len(self.jobs) > 0:
             if self.is_shutdown():
                 self.scancel_jobs()
@@ -581,19 +570,22 @@ class RunnerSlurmSudo(BaseRunner):
                     result, exception = self._postprocess_single_task(
                         task=slurm_job.tasks[0]
                     )
-                    if result is not None:
-                        if task_type not in ["compound", "converter_compound"]:
-                            update_status_of_history_unit(
-                                history_unit_id=history_unit_id,
-                                status=HistoryUnitStatus.DONE,
-                                db_sync=db,
-                            )
+                    # Note: the relevant done/failed check is based on
+                    # whether `exception is None`. The fact that
+                    # `result is None` is not relevant for this purpose.
                     if exception is not None:
                         update_status_of_history_unit(
                             history_unit_id=history_unit_id,
                             status=HistoryUnitStatus.FAILED,
                             db_sync=db,
                         )
+                    else:
+                        if task_type not in ["compound", "converter_compound"]:
+                            update_status_of_history_unit(
+                                history_unit_id=history_unit_id,
+                                status=HistoryUnitStatus.DONE,
+                                db_sync=db,
+                            )
 
             time.sleep(self.slurm_poll_interval)
 
@@ -610,21 +602,8 @@ class RunnerSlurmSudo(BaseRunner):
     ):
 
         if len(self.jobs) > 0:
-            raise RuntimeError(f"Cannot run .submit when {len(self.jobs)=}")
-
-        if task_type in ["compound", "converter_compound"]:
-            if len(history_unit_ids) != 1:
-                raise NotImplementedError(
-                    "We are breaking the assumption that compound/multisubmit "
-                    "is associated to a single HistoryUnit. This is not "
-                    "supported."
-                )
-        elif task_type == "parallel" and len(history_unit_ids) != len(
-            list_parameters
-        ):
-            raise ValueError(
-                f"{len(history_unit_ids)=} differs from "
-                f"{len(list_parameters)=}."
+            raise RuntimeError(
+                f"Cannot run .multisubmit when {len(self.jobs)=}"
             )
 
         self.validate_multisubmit_parameters(
@@ -632,12 +611,19 @@ class RunnerSlurmSudo(BaseRunner):
             task_type=task_type,
             list_task_files=list_task_files,
         )
+        self.validate_multisubmit_history_unit_ids(
+            history_unit_ids=history_unit_ids,
+            task_type=task_type,
+            list_parameters=list_parameters,
+        )
+
+        logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
 
         workdir_local = list_task_files[0].wftask_subfolder_local
         workdir_remote = list_task_files[0].wftask_subfolder_remote
 
         # Create local&remote task subfolders
-        if task_type not in ["converter_compound", "compound"]:
+        if task_type == "parallel":
             original_umask = os.umask(0)
             workdir_local.mkdir(parents=True, mode=0o755)
             os.umask(original_umask)
@@ -714,18 +700,14 @@ class RunnerSlurmSudo(BaseRunner):
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
-        logger.info(f"END submission phase, {list(self.jobs.keys())=}")
+        logger.info(f"END submission phase, {self.job_ids=}")
 
-        # FIXME
-        jobs_that_started = set()
-        while len(jobs_that_started) != len(self.job_ids):
-            res = run_squeue(self.job_ids)
-            new_jobs = set(out.split()[0] for out in res.stdout.splitlines())
-            jobs_that_started = jobs_that_started.union(new_jobs)
-            logger.debug(f"{new_jobs=}")
-            logger.debug(f"{len(jobs_that_started)=}")
+        # FIXME: Replace with more robust/efficient logic
+        logger.warning("Now sleep 4 (FIXME)")
+        time.sleep(4)
 
         # Retrieval phase
+        logger.info("START retrieval phase")
         while len(self.jobs) > 0:
             if self.is_shutdown():
                 self.scancel_jobs()
@@ -737,28 +719,38 @@ class RunnerSlurmSudo(BaseRunner):
                     slurm_job = self.jobs.pop(slurm_job_id)
                     self._copy_files_from_remote_to_local(slurm_job)
                     for task in slurm_job.tasks:
+                        logger.debug(f"Now processing {task.index=}")
                         result, exception = self._postprocess_single_task(
                             task=task
                         )
 
-                        if result is not None:
-                            results[task.index] = result
+                        # Note: the relevant done/failed check is based on
+                        # whether `exception is None`. The fact that
+                        # `result is None` is not relevant for this purpose.
+                        if exception is not None:
+                            logger.debug(
+                                f"Task {task.index} has an exception."
+                            )  # FIXME  # noqa
+                            exceptions[task.index] = exception
                             if task_type == "parallel":
                                 update_status_of_history_unit(
                                     history_unit_id=history_unit_ids[
                                         task.index
                                     ],
-                                    status=HistoryUnitStatus.DONE,
+                                    status=HistoryUnitStatus.FAILED,
                                     db_sync=db,
                                 )
-                        if exception is not None:
-                            exceptions[task.index] = exception
+                        else:
+                            logger.debug(
+                                f"Task {task.index} has no exception."
+                            )  # FIXME  # noqa
+                            results[task.index] = result
                             if task_type == "parallel":
                                 update_status_of_history_unit(
                                     history_unit_id=history_unit_ids[
                                         task.index
                                     ],
-                                    status=HistoryUnitStatus.FAILED,
+                                    status=HistoryUnitStatus.DONE,
                                     db_sync=db,
                                 )
 

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/fractal_server/app/runner/v2/db_tools.py

@@ -85,3 +85,4 @@ def bulk_upsert_image_cache_fast(
             ),
         )
         db.execute(stmt)
+        db.commit()

{fractal_server-2.14.0a11 → fractal_server-2.14.0a13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a11"
+version = "2.14.0a13"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a11"
+current_version = "2.14.0a13"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a11/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a11"