fractal-server 2.14.0a19__py3-none-any.whl → 2.14.0a21__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a19"
+__VERSION__ = "2.14.0a21"

fractal_server/app/routes/api/v2/history.py

@@ -41,6 +41,24 @@ from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
 
+def check_historyrun_related_to_dataset_and_wftask(
+    history_run: HistoryRun,
+    dataset_id: int,
+    workflowtask_id: int,
+):
+    if (
+        history_run.dataset_id != dataset_id
+        or history_run.workflowtask_id != workflowtask_id
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Invalid query parameters: HistoryRun[{history_run.id}] is "
+                f"not related to {dataset_id=} and {workflowtask_id=}."
+            ),
+        )
+
+
 class ImageWithStatusPage(PaginationResponse[SingleImageWithStatus]):
 
     attributes: dict[str, list[Any]]
@@ -199,7 +217,14 @@ async def get_history_run_units(
     )
 
     # Check that `HistoryRun` exists
-    await get_history_run_or_404(history_run_id=history_run_id, db=db)
+    history_run = await get_history_run_or_404(
+        history_run_id=history_run_id, db=db
+    )
+    check_historyrun_related_to_dataset_and_wftask(
+        history_run=history_run,
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
+    )
 
     # Count `HistoryUnit`s
     stmt = select(func.count(HistoryUnit.id)).where(
@@ -450,6 +475,23 @@ async def get_history_unit_log(
         db=db,
     )
 
+    if history_unit.history_run_id != history_run_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Invalid query parameters: HistoryUnit[{history_unit_id}] "
+                f"is not related to HistoryRun[{history_run_id}]"
+            ),
+        )
+    history_run = await get_history_run_or_404(
+        history_run_id=history_run_id, db=db
+    )
+    check_historyrun_related_to_dataset_and_wftask(
+        history_run=history_run,
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
+    )
+
     # Get log or placeholder text
     log = read_log_file(
         logfile=history_unit.logfile,

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,6 +23,9 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    bulk_update_status_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import (
     update_logfile_of_history_unit,
 )
@@ -47,6 +50,7 @@ class BaseSlurmRunner(BaseRunner):
     root_dir_local: Path
     root_dir_remote: Path
     poll_interval: int
+    poll_interval_internal: float
     jobs: dict[str, SlurmJob]
     python_worker_interpreter: str
     slurm_runner_type: Literal["ssh", "sudo"]
@@ -72,6 +76,8 @@ class BaseSlurmRunner(BaseRunner):
         self.poll_interval = (
             poll_interval or settings.FRACTAL_SLURM_POLL_INTERVAL
         )
+        self.poll_interval_internal = self.poll_interval / 10.0
+
         self.check_fractal_server_versions()
 
         # Create job folders. Note that the local one may or may not exist
@@ -405,6 +411,33 @@ class BaseSlurmRunner(BaseRunner):
     def job_ids(self) -> list[str]:
         return list(self.jobs.keys())
 
+    def wait_and_check_shutdown(self) -> list[str]:
+        """
+        Wait at most `self.poll_interval`, while also checking for shutdown.
+        """
+        # Sleep for `self.poll_interval`, but keep checking for shutdowns
+        start_time = time.perf_counter()
+        max_time = start_time + self.poll_interval
+        can_return = False
+        logger.debug(
+            "[wait_and_check_shutdown] "
+            f"I will wait at most {self.poll_interval} s, "
+            f"in blocks of {self.poll_interval_internal} s."
+        )
+
+        while (time.perf_counter() < max_time) or (can_return is False):
+            # Handle shutdown
+            if self.is_shutdown():
+                logger.info("[wait_and_check_shutdown] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[wait_and_check_shutdown] {scancelled_job_ids=}")
+                return scancelled_job_ids
+            can_return = True
+            time.sleep(self.poll_interval_internal)
+
+        logger.debug("[wait_and_check_shutdown] No shutdown file detected")
+        return []
+
     def submit(
         self,
         func: callable,
@@ -491,15 +524,9 @@ class BaseSlurmRunner(BaseRunner):
 
         # Retrieval phase
         logger.info("[submit] START retrieval phase")
+        scancelled_job_ids = []
         while len(self.jobs) > 0:
 
-            # Handle shutdown
-            scancelled_job_ids = []
-            if self.is_shutdown():
-                logger.info("[submit] Shutdown file detected")
-                scancelled_job_ids = self.scancel_jobs()
-                logger.info(f"[submit] {scancelled_job_ids=}")
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[submit] {finished_job_ids=}")
@@ -531,7 +558,8 @@ class BaseSlurmRunner(BaseRunner):
                                 db_sync=db,
                             )
 
-            time.sleep(self.poll_interval)
+            if len(self.jobs) > 0:
+                scancelled_job_ids = self.wait_and_check_shutdown()
 
         logger.info("[submit] END")
         return result, exception
@@ -554,13 +582,11 @@ class BaseSlurmRunner(BaseRunner):
         if self.is_shutdown():
             if task_type == "parallel":
                 with next(get_sync_db()) as db:
-                    # FIXME: Replace with bulk function
-                    for history_unit_id in history_unit_ids:
-                        update_status_of_history_unit(
-                            history_unit_id=history_unit_id,
-                            status=HistoryUnitStatus.FAILED,
-                            db_sync=db,
-                        )
+                    bulk_update_status_of_history_unit(
+                        history_unit_ids=history_unit_ids,
+                        status=HistoryUnitStatus.FAILED,
+                        db_sync=db,
+                    )
             results = {}
             exceptions = {
                 ind: SHUTDOWN_EXCEPTION for ind in range(len(list_parameters))
@@ -687,17 +713,11 @@ class BaseSlurmRunner(BaseRunner):
         logger.info("[multisubmit] START retrieval phase")
         while len(self.jobs) > 0:
 
-            # Handle shutdown
-            scancelled_job_ids = []
-            if self.is_shutdown():
-                logger.info("[multisubmit] Shutdown file detected")
-                scancelled_job_ids = self.scancel_jobs()
-                logger.info(f"[multisubmit] {scancelled_job_ids=}")
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[multisubmit] {finished_job_ids=}")
 
+            scancelled_job_ids = []
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
                     logger.info(f"[multisubmit] Now process {slurm_job_id=}")
@@ -737,7 +757,10 @@ class BaseSlurmRunner(BaseRunner):
                                     db_sync=db,
                                 )
 
-            time.sleep(self.poll_interval)
+            if len(self.jobs) > 0:
+                scancelled_job_ids = self.wait_and_check_shutdown()
+
+        logger.info("[multisubmit] END")
         return results, exceptions
 
     def check_fractal_server_versions(self) -> None:

fractal_server/app/runner/v2/db_tools.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
+from sqlmodel import update
 
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
@@ -40,6 +41,24 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
+def bulk_update_status_of_history_unit(
+    *,
+    history_unit_ids: list[int],
+    status: HistoryUnitStatus,
+    db_sync: Session,
+) -> None:
+    for ind in range(0, len(history_unit_ids), _CHUNK_SIZE):
+        db_sync.execute(
+            update(HistoryUnit)
+            .where(
+                HistoryUnit.id.in_(history_unit_ids[ind : ind + _CHUNK_SIZE])
+            )
+            .values(status=status)
+        )
+        # NOTE: keeping commit within the for loop is much more efficient
+        db_sync.commit()
+
+
 def update_logfile_of_history_unit(
     *,
     history_unit_id: int,

fractal_server/app/runner/v2/runner.py

@@ -8,6 +8,7 @@ from typing import Literal
 from typing import Optional
 
 from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import delete
 
 from ....images import SingleImage
 from ....images.tools import filter_image_list
@@ -21,6 +22,7 @@ from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
@@ -331,6 +333,18 @@ def execute_tasks_v2(
             db_dataset.images = tmp_images
             flag_modified(db_dataset, "images")
             db.merge(db_dataset)
+
+            db.execute(
+                delete(HistoryImageCache)
+                .where(HistoryImageCache.dataset_id == dataset.id)
+                .where(HistoryImageCache.workflowtask_id == wftask.id)
+                .where(
+                    HistoryImageCache.zarr_url.in_(
+                        current_task_output.image_list_removals
+                    )
+                )
+            )
+
             db.commit()
             db.close()  # FIXME: why is this needed?
 

{fractal_server-2.14.0a19.dist-info → fractal_server-2.14.0a21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a19
+Version: 2.14.0a21
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.0a19.dist-info → fractal_server-2.14.0a21.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=pyEYIRGyt1YpfP5AuqBuQLMiN6pJi7BgKIMeFoganSM,26
+fractal_server/__init__.py,sha256=X5Dy_f87GBiFeLzs2riLgudM2HP43U0ZuXNsU2NF7Os,26
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,7 +36,7 @@ fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=ZlI6nwzB5r9AiY
 fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=qdXCb6IP8-qPEAxGZKljtjIqNzIAyRaAsQSRi5VqFHM,6773
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=uhNSs-jcS7ndIUFKiOC1yrDiViw3uvKEXi9UL04BMks,11642
 fractal_server/app/routes/api/v2/dataset.py,sha256=h5AhE0sdhQ20ZlIbEJsFnHIOUW0S1VHFpoflpBkVScs,8936
-fractal_server/app/routes/api/v2/history.py,sha256=BuXVCDUCIYG5YF49D4b_CsqQrx6uFH6WB4TgkrdS2zo,15926
+fractal_server/app/routes/api/v2/history.py,sha256=lMbaybooBzzbCgD9vdzPyNxdgAZuzCH_YrW9ost-UgI,17253
 fractal_server/app/routes/api/v2/images.py,sha256=BGpO94gVd8BTpCN6Mun2RXmjrPmfkIp73m8RN7uiGW4,8361
 fractal_server/app/routes/api/v2/job.py,sha256=MU1sHIKk_89WrD0TD44d4ufzqnywot7On_W71KjyUbQ,6500
 fractal_server/app/routes/api/v2/project.py,sha256=uAZgATiHcOvbnRX-vv1D3HoaEUvLUd7vzVmGcqOP8ZY,4602
@@ -78,7 +78,7 @@ fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=fZaFUUXqDH0p3DndCFUpFqTqyD2tMVCuSYgYLAycpVw,15897
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=gyarQieNwmxoy1PnPSjEEcU-VXBI1IHADF66EoZioTA,30270
+fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=f01IDAtBoatOYEP4UtrH0Y4qN7BwM1ov4Bx8rotQg1M,31099
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=-fAX1DZMB5RZnyYanIJD72mWOJAPkh21jd4loDXKJw4,5994
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=FS_F8EaPp-A5eQT5_ZH3ICCHt0-C8b_2OSYcyRkXnb4,5851
 fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=YGgzTspkK9ItSMzwuYv_1tY7_1g89Qpeny5Auinxk1E,2708
@@ -98,10 +98,10 @@ fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 fractal_server/app/runner/v2/_local.py,sha256=DK8yagbvd6HHjcDVhUzTy0f7MURlTkQha-NM6OZKgJc,3044
 fractal_server/app/runner/v2/_slurm_ssh.py,sha256=_bytOf8z9sdrhI03D6eqg-aQPnJ7V2-qnqpcHAYizns,3278
 fractal_server/app/runner/v2/_slurm_sudo.py,sha256=DBCNxifXmMkpu71Wnk5u9-wKT7PV1WROQuY_4DYoZRI,2993
-fractal_server/app/runner/v2/db_tools.py,sha256=tyLGY-g4ISZSJzk6ootEuHocdVwMg3taBL0oIivXV7M,2846
+fractal_server/app/runner/v2/db_tools.py,sha256=KoPpbUtY5a1M4__Pp4luN8_TkpdkaUZXH_lkNoXHu4s,3394
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
-fractal_server/app/runner/v2/runner.py,sha256=SsKEZAsB8sPV8W3khTkAqaGdDwoTm_lav-fx6DdCwyA,15294
+fractal_server/app/runner/v2/runner.py,sha256=4Wrtx0HjjWD92yuIHqIxywPv8DeDh4iR9dZHXozIysQ,15781
 fractal_server/app/runner/v2/runner_functions.py,sha256=5cK5O2rTrCsCxMTVN3iNPRwZ_891BC9_RMo64a8ZGYw,16338
 fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=9t1CHN3EyfsGRWfG257YPY5WjQ6zuztsw_KZrpEAFPo,3703
 fractal_server/app/runner/v2/submit_workflow.py,sha256=EDUyUuIPwZHb2zm7SCRRoFsGq2cN-b5OKw6CYkZ8kWk,13048
@@ -207,8 +207,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENP
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a19.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a19.dist-info/METADATA,sha256=nHAM5HDPXZ3Y5Nq5z7yU-DyILe3YESFIZdQYezpy4Xc,4563
-fractal_server-2.14.0a19.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.0a19.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a19.dist-info/RECORD,,
+fractal_server-2.14.0a21.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a21.dist-info/METADATA,sha256=OZesW99axIUpT41CcJOkIszG_F5jLrDfKLT29EctIgI,4563
+fractal_server-2.14.0a21.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.0a21.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a21.dist-info/RECORD,,