fractal-server 2.14.0a20__tar.gz → 2.14.0a22__tar.gz

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a20
+Version: 2.14.0a22
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a22/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a22"

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/fractal_server/app/routes/api/v2/history.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 from typing import Any
 from typing import Optional
 
@@ -41,6 +42,24 @@ from fractal_server.images.tools import merge_type_filters
 from fractal_server.logger import set_logger
 
 
+def check_historyrun_related_to_dataset_and_wftask(
+    history_run: HistoryRun,
+    dataset_id: int,
+    workflowtask_id: int,
+):
+    if (
+        history_run.dataset_id != dataset_id
+        or history_run.workflowtask_id != workflowtask_id
+    ):
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Invalid query parameters: HistoryRun[{history_run.id}] is "
+                f"not related to {dataset_id=} and {workflowtask_id=}."
+            ),
+        )
+
+
 class ImageWithStatusPage(PaginationResponse[SingleImageWithStatus]):
 
     attributes: dict[str, list[Any]]
@@ -74,7 +93,7 @@ async def get_workflow_tasks_statuses(
         db=db,
     )
 
-    response = {}
+    response: dict[int, dict[str, int | str] | None] = {}
     for wftask in workflow.task_list:
         res = await db.execute(
             select(HistoryRun)
@@ -112,7 +131,18 @@ async def get_workflow_tasks_statuses(
                 f"num_{target_status.value}_images"
             ] = num_images
 
-    return JSONResponse(content=response, status_code=200)
+    new_response = deepcopy(response)
+    for key, value in response.items():
+        if value is not None:
+            num_total_images = sum(
+                value[f"num_{target_status.value}_images"]
+                for target_status in HistoryUnitStatus
+            )
+            if num_total_images > value["num_available_images"]:
+                value["num_available_images"] = None
+        new_response[key] = value
+
+    return JSONResponse(content=new_response, status_code=200)
 
 
 @router.get("/project/{project_id}/status/run/")
@@ -199,7 +229,14 @@ async def get_history_run_units(
     )
 
     # Check that `HistoryRun` exists
-    await get_history_run_or_404(history_run_id=history_run_id, db=db)
+    history_run = await get_history_run_or_404(
+        history_run_id=history_run_id, db=db
+    )
+    check_historyrun_related_to_dataset_and_wftask(
+        history_run=history_run,
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
+    )
 
     # Count `HistoryUnit`s
     stmt = select(func.count(HistoryUnit.id)).where(
@@ -450,6 +487,23 @@ async def get_history_unit_log(
         db=db,
     )
 
+    if history_unit.history_run_id != history_run_id:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(
+                f"Invalid query parameters: HistoryUnit[{history_unit_id}] "
+                f"is not related to HistoryRun[{history_run_id}]"
+            ),
+        )
+    history_run = await get_history_run_or_404(
+        history_run_id=history_run_id, db=db
+    )
+    check_historyrun_related_to_dataset_and_wftask(
+        history_run=history_run,
+        dataset_id=dataset_id,
+        workflowtask_id=workflowtask_id,
+    )
+
     # Get log or placeholder text
     log = read_log_file(
         logfile=history_unit.logfile,

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/fractal_server/app/runner/compress_folder.py

@@ -11,8 +11,8 @@ built-in `tarfile` library has to do with performance issues we observed
 when handling files which were just created within a SLURM job, and in the
 context of a CephFS filesystem.
 """
-import shutil
 import sys
+import time
 from pathlib import Path
 
 from fractal_server.app.runner.run_subprocess import run_subprocess
@@ -20,21 +20,26 @@ from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
 
 
-def copy_subfolder(src: Path, dest: Path, logger_name: str):
+def _copy_subfolder(src: Path, dest: Path, logger_name: str):
+    t_start = time.perf_counter()
     cmd_cp = f"cp -r {src.as_posix()} {dest.as_posix()}"
     logger = get_logger(logger_name=logger_name)
     logger.debug(f"{cmd_cp=}")
     res = run_subprocess(cmd=cmd_cp, logger_name=logger_name)
+    elapsed = time.perf_counter() - t_start
+    logger.debug(f"[_copy_subfolder] END {elapsed=} s ({dest.as_posix()})")
     return res
 
 
-def create_tar_archive(
-    tarfile_path: Path,
+def _create_tar_archive(
+    tarfile_path: str,
     subfolder_path_tmp_copy: Path,
     logger_name: str,
     remote_to_local: bool,
 ):
     logger = get_logger(logger_name)
+    logger.debug(f"[_create_tar_archive] START ({tarfile_path})")
+    t_start = time.perf_counter()
 
     if remote_to_local:
         exclude_options = "--exclude *sbatch --exclude *_in_*.pickle "
@@ -49,15 +54,24 @@ def create_tar_archive(
     )
     logger.debug(f"cmd tar:\n{cmd_tar}")
     run_subprocess(cmd=cmd_tar, logger_name=logger_name, allow_char="*")
+    elapsed = time.perf_counter() - t_start
+    logger.debug(f"[_create_tar_archive] END {elapsed=} s ({tarfile_path})")
 
 
-def remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
+def _remove_temp_subfolder(subfolder_path_tmp_copy: Path, logger_name: str):
     logger = get_logger(logger_name)
+    t_start = time.perf_counter()
     try:
-        logger.debug(f"Now remove {subfolder_path_tmp_copy}")
-        shutil.rmtree(subfolder_path_tmp_copy)
+        cmd_rm = f"rm -rf {subfolder_path_tmp_copy}"
+        logger.debug(f"cmd rm:\n{cmd_rm}")
+        run_subprocess(cmd=cmd_rm, logger_name=logger_name, allow_char="*")
     except Exception as e:
-        logger.debug(f"ERROR during shutil.rmtree: {e}")
+        logger.debug(f"ERROR during {cmd_rm}: {e}")
+    elapsed = time.perf_counter() - t_start
+    logger.debug(
+        f"[_remove_temp_subfolder] END {elapsed=} s "
+        f"({subfolder_path_tmp_copy=})"
+    )
 
 
 def compress_folder(
@@ -91,10 +105,12 @@ def compress_folder(
         subfolder_path.parent / f"{subfolder_path.name}_copy"
     )
     try:
-        copy_subfolder(
-            subfolder_path, subfolder_path_tmp_copy, logger_name=logger_name
+        _copy_subfolder(
+            subfolder_path,
+            subfolder_path_tmp_copy,
+            logger_name=logger_name,
         )
-        create_tar_archive(
+        _create_tar_archive(
             tarfile_path,
             subfolder_path_tmp_copy,
             logger_name=logger_name,
@@ -107,7 +123,9 @@ def compress_folder(
         sys.exit(1)
 
     finally:
-        remove_temp_subfolder(subfolder_path_tmp_copy, logger_name=logger_name)
+        _remove_temp_subfolder(
+            subfolder_path_tmp_copy, logger_name=logger_name
+        )
 
 
 def main(sys_argv: list[str]):

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,6 +23,9 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    bulk_update_status_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import (
     update_logfile_of_history_unit,
 )
@@ -47,6 +50,7 @@ class BaseSlurmRunner(BaseRunner):
     root_dir_local: Path
     root_dir_remote: Path
     poll_interval: int
+    poll_interval_internal: float
     jobs: dict[str, SlurmJob]
     python_worker_interpreter: str
     slurm_runner_type: Literal["ssh", "sudo"]
@@ -72,6 +76,8 @@ class BaseSlurmRunner(BaseRunner):
         self.poll_interval = (
             poll_interval or settings.FRACTAL_SLURM_POLL_INTERVAL
         )
+        self.poll_interval_internal = self.poll_interval / 10.0
+
         self.check_fractal_server_versions()
 
         # Create job folders. Note that the local one may or may not exist
@@ -405,6 +411,33 @@ class BaseSlurmRunner(BaseRunner):
     def job_ids(self) -> list[str]:
         return list(self.jobs.keys())
 
+    def wait_and_check_shutdown(self) -> list[str]:
+        """
+        Wait at most `self.poll_interval`, while also checking for shutdown.
+        """
+        # Sleep for `self.poll_interval`, but keep checking for shutdowns
+        start_time = time.perf_counter()
+        max_time = start_time + self.poll_interval
+        can_return = False
+        logger.debug(
+            "[wait_and_check_shutdown] "
+            f"I will wait at most {self.poll_interval} s, "
+            f"in blocks of {self.poll_interval_internal} s."
+        )
+
+        while (time.perf_counter() < max_time) or (can_return is False):
+            # Handle shutdown
+            if self.is_shutdown():
+                logger.info("[wait_and_check_shutdown] Shutdown file detected")
+                scancelled_job_ids = self.scancel_jobs()
+                logger.info(f"[wait_and_check_shutdown] {scancelled_job_ids=}")
+                return scancelled_job_ids
+            can_return = True
+            time.sleep(self.poll_interval_internal)
+
+        logger.debug("[wait_and_check_shutdown] No shutdown file detected")
+        return []
+
     def submit(
         self,
         func: callable,
@@ -491,15 +524,9 @@ class BaseSlurmRunner(BaseRunner):
 
         # Retrieval phase
         logger.info("[submit] START retrieval phase")
+        scancelled_job_ids = []
         while len(self.jobs) > 0:
 
-            # Handle shutdown
-            scancelled_job_ids = []
-            if self.is_shutdown():
-                logger.info("[submit] Shutdown file detected")
-                scancelled_job_ids = self.scancel_jobs()
-                logger.info(f"[submit] {scancelled_job_ids=}")
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[submit] {finished_job_ids=}")
@@ -532,7 +559,7 @@ class BaseSlurmRunner(BaseRunner):
                             )
 
             if len(self.jobs) > 0:
-                time.sleep(self.poll_interval)
+                scancelled_job_ids = self.wait_and_check_shutdown()
 
         logger.info("[submit] END")
         return result, exception
@@ -555,13 +582,11 @@ class BaseSlurmRunner(BaseRunner):
         if self.is_shutdown():
             if task_type == "parallel":
                 with next(get_sync_db()) as db:
-                    # FIXME: Replace with bulk function
-                    for history_unit_id in history_unit_ids:
-                        update_status_of_history_unit(
-                            history_unit_id=history_unit_id,
-                            status=HistoryUnitStatus.FAILED,
-                            db_sync=db,
-                        )
+                    bulk_update_status_of_history_unit(
+                        history_unit_ids=history_unit_ids,
+                        status=HistoryUnitStatus.FAILED,
+                        db_sync=db,
+                    )
             results = {}
             exceptions = {
                 ind: SHUTDOWN_EXCEPTION for ind in range(len(list_parameters))
@@ -688,17 +713,11 @@ class BaseSlurmRunner(BaseRunner):
         logger.info("[multisubmit] START retrieval phase")
         while len(self.jobs) > 0:
 
-            # Handle shutdown
-            scancelled_job_ids = []
-            if self.is_shutdown():
-                logger.info("[multisubmit] Shutdown file detected")
-                scancelled_job_ids = self.scancel_jobs()
-                logger.info(f"[multisubmit] {scancelled_job_ids=}")
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[multisubmit] {finished_job_ids=}")
 
+            scancelled_job_ids = []
             with next(get_sync_db()) as db:
                 for slurm_job_id in finished_job_ids:
                     logger.info(f"[multisubmit] Now process {slurm_job_id=}")
@@ -738,7 +757,10 @@ class BaseSlurmRunner(BaseRunner):
                                     db_sync=db,
                                 )
 
-            time.sleep(self.poll_interval)
+            if len(self.jobs) > 0:
+                scancelled_job_ids = self.wait_and_check_shutdown()
+
+        logger.info("[multisubmit] END")
         return results, exceptions
 
     def check_fractal_server_versions(self) -> None:

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/fractal_server/app/runner/v2/db_tools.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
+from sqlmodel import update
 
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
@@ -40,6 +41,24 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
+def bulk_update_status_of_history_unit(
+    *,
+    history_unit_ids: list[int],
+    status: HistoryUnitStatus,
+    db_sync: Session,
+) -> None:
+    for ind in range(0, len(history_unit_ids), _CHUNK_SIZE):
+        db_sync.execute(
+            update(HistoryUnit)
+            .where(
+                HistoryUnit.id.in_(history_unit_ids[ind : ind + _CHUNK_SIZE])
+            )
+            .values(status=status)
+        )
+        # NOTE: keeping commit within the for loop is much more efficient
+        db_sync.commit()
+
+
 def update_logfile_of_history_unit(
     *,
     history_unit_id: int,

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/fractal_server/app/runner/v2/runner.py

@@ -8,6 +8,7 @@ from typing import Literal
 from typing import Optional
 
 from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import delete
 
 from ....images import SingleImage
 from ....images.tools import filter_image_list
@@ -21,6 +22,7 @@ from .task_interface import TaskOutput
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import AccountingRecord
 from fractal_server.app.models.v2 import DatasetV2
+from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
@@ -331,6 +333,18 @@ def execute_tasks_v2(
             db_dataset.images = tmp_images
             flag_modified(db_dataset, "images")
             db.merge(db_dataset)
+
+            db.execute(
+                delete(HistoryImageCache)
+                .where(HistoryImageCache.dataset_id == dataset.id)
+                .where(HistoryImageCache.workflowtask_id == wftask.id)
+                .where(
+                    HistoryImageCache.zarr_url.in_(
+                        current_task_output.image_list_removals
+                    )
+                )
+            )
+
             db.commit()
             db.close()  # FIXME: why is this needed?
 

{fractal_server-2.14.0a20 → fractal_server-2.14.0a22}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a20"
+version = "2.14.0a22"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a20"
+current_version = "2.14.0a22"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a20/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a20"