fractal-server 2.14.0a24__tar.gz → 2.14.0a25__tar.gz

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a24
+Version: 2.14.0a25
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a25/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a25"

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/models/v2/history.py

@@ -50,7 +50,7 @@ class HistoryUnit(SQLModel, table=True):
         ondelete="CASCADE",
     )
 
-    logfile: Optional[str]
+    logfile: str
     status: str
     zarr_urls: list[str] = Field(
         sa_column=Column(ARRAY(String)),

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/routes/api/v2/history.py

@@ -249,8 +249,10 @@ async def get_history_run_units(
     page_size = pagination.page_size or total_count
 
     # Query `HistoryUnit`s
-    stmt = select(HistoryUnit).where(
-        HistoryUnit.history_run_id == history_run_id
+    stmt = (
+        select(HistoryUnit)
+        .where(HistoryUnit.history_run_id == history_run_id)
+        .order_by(HistoryUnit.id)
     )
     if unit_status:
         stmt = stmt.where(HistoryUnit.status == unit_status)

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/base_runner.py

@@ -4,7 +4,6 @@ from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.app.schemas.v2.task import TaskTypeType
 from fractal_server.logger import set_logger
 
-
 TASK_TYPES_SUBMIT: list[TaskTypeType] = [
     "compound",
     "converter_compound",
@@ -103,29 +102,44 @@ class BaseRunner(object):
 
     def validate_multisubmit_parameters(
         self,
-        list_parameters: list[dict[str, Any]],
+        *,
         task_type: TaskTypeType,
+        list_parameters: list[dict[str, Any]],
         list_task_files: list[TaskFiles],
+        history_unit_ids: list[int],
     ) -> None:
         """
         Validate parameters for `multisubmit` method
 
         Args:
-            list_parameters: List of parameters dictionaries.
             task_type: Task type.
+            list_parameters: List of parameters dictionaries.
+            list_task_files:
+            history_unit_ids:
         """
         if task_type not in TASK_TYPES_MULTISUBMIT:
             raise ValueError(f"Invalid {task_type=} for `multisubmit`.")
 
+        if not isinstance(list_parameters, list):
+            raise ValueError("`parameters` must be a list.")
+
+        if len(list_parameters) != len(list_task_files):
+            raise ValueError(
+                f"{len(list_task_files)=} differs from "
+                f"{len(list_parameters)=}."
+            )
+        if len(history_unit_ids) != len(list_parameters):
+            raise ValueError(
+                f"{len(history_unit_ids)=} differs from "
+                f"{len(list_parameters)=}."
+            )
+
         subfolders = set(
             task_file.wftask_subfolder_local for task_file in list_task_files
         )
         if len(subfolders) != 1:
             raise ValueError(f"More than one subfolders: {subfolders}.")
 
-        if not isinstance(list_parameters, list):
-            raise ValueError("`parameters` must be a list.")
-
         for single_kwargs in list_parameters:
             if not isinstance(single_kwargs, dict):
                 raise ValueError("kwargs itemt must be a dictionary.")
@@ -137,33 +151,3 @@ class BaseRunner(object):
             zarr_urls = [kwargs["zarr_url"] for kwargs in list_parameters]
             if len(zarr_urls) != len(set(zarr_urls)):
                 raise ValueError("Non-unique zarr_urls")
-
-    def validate_multisubmit_history_unit_ids(
-        self,
-        *,
-        history_unit_ids: list[int],
-        task_type: TaskTypeType,
-        list_parameters: list[dict[str, Any]],
-    ) -> None:
-        """
-        Run preliminary check for multisubmit inputs.
-
-        Args:
-            history_unit_ids:
-            task_type:
-            list_parameters:
-        """
-        if task_type in ["compound", "converter_compound"]:
-            if len(history_unit_ids) != 1:
-                raise NotImplementedError(
-                    "We are breaking the assumption that compound/multisubmit "
-                    "is associated to a single HistoryUnit. This is not "
-                    "supported."
-                )
-            elif task_type == "parallel" and len(history_unit_ids) != len(
-                list_parameters
-            ):
-                raise ValueError(
-                    f"{len(history_unit_ids)=} differs from "
-                    f"{len(list_parameters)=}."
-                )

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/local/get_local_config.py

@@ -47,11 +47,16 @@ class LocalBackendConfig(BaseModel):
     model_config = ConfigDict(extra="forbid")
     parallel_tasks_per_job: Optional[int] = None
 
+    @property
+    def batch_size(self) -> int:
+        return self.parallel_tasks_per_job or 1
+
 
 def get_local_backend_config(
     wftask: WorkflowTaskV2,
     which_type: Literal["non_parallel", "parallel"],
     config_path: Optional[Path] = None,
+    tot_tasks: int = 1,
 ) -> LocalBackendConfig:
     """
     Prepare a `LocalBackendConfig` configuration object

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/local/runner.py

@@ -8,12 +8,7 @@ from .get_local_config import LocalBackendConfig
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
-from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
-from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
-from fractal_server.app.runner.v2.db_tools import (
-    update_logfile_of_history_unit,
-)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -68,13 +63,6 @@ class LocalRunner(BaseRunner):
         workdir_local = task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
-        # Add prefix to task_files object
-        task_files.prefix = SUBMIT_PREFIX
-        update_logfile_of_history_unit(
-            history_unit_id=history_unit_id,
-            logfile=task_files.log_file_local,
-        )
-
         # SUBMISSION PHASE
         future = self.executor.submit(
             func,
@@ -111,29 +99,18 @@ class LocalRunner(BaseRunner):
         list_task_files: list[TaskFiles],
         task_type: Literal["parallel", "compound", "converter_compound"],
         config: LocalBackendConfig,
-    ):
+    ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
-        Note:
-
-        1. The number of sruns and futures is equal to `len(list_parameters)`.
-        2. The number of `HistoryUnit`s is equal to `len(history_unit_ids)`.
-        3. For compound tasks, these two numbers are not the same.
-
-        For this reason, we defer database updates to the caller function,
-        when we are in one of the "compound" cases
-
+        Note: `list_parameters`, `list_task_files` and `history_unit_ids`
+        have the same size. For parallel tasks, this is also the number of
+        input images, while for compound tasks these can differ.
         """
 
         self.validate_multisubmit_parameters(
             list_parameters=list_parameters,
             task_type=task_type,
             list_task_files=list_task_files,
-        )
-
-        self.validate_multisubmit_history_unit_ids(
             history_unit_ids=history_unit_ids,
-            task_type=task_type,
-            list_parameters=list_parameters,
         )
 
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
@@ -159,9 +136,6 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                list_task_files[
-                    positional_index
-                ].prefix = f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
                 future = self.executor.submit(
                     func,
                     parameters=kwargs,
@@ -171,23 +145,6 @@ class LocalRunner(BaseRunner):
                 )
                 active_futures[positional_index] = future
 
-                if task_type == "parallel":
-                    # FIXME: replace loop with a `bulk_update_history_unit`
-                    # function
-                    update_logfile_of_history_unit(
-                        history_unit_id=history_unit_ids[positional_index],
-                        logfile=list_task_files[
-                            positional_index
-                        ].log_file_local,
-                    )
-                else:
-                    logger.debug(
-                        f"Unclear what logfile to associate to {task_type=} "
-                        "within multisubmit (see issue #2382)."
-                    )
-                    # FIXME: Improve definition for compound tasks
-                    pass
-
             while active_futures:
                 finished_futures = [
                     index_and_future

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -407,6 +407,10 @@ class SlurmConfig(BaseModel):
 
         return lines
 
+    @property
+    def batch_size(self) -> int:
+        return self.tasks_per_job
+
 
 def _parse_mem_value(raw_mem: Union[str, int]) -> int:
     """

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -12,7 +12,6 @@ import cloudpickle
 from ..slurm_common._slurm_config import SlurmConfig
 from ..slurm_common.slurm_job_task_models import SlurmJob
 from ..slurm_common.slurm_job_task_models import SlurmTask
-from ._batching import heuristics
 from ._job_states import STATES_FINISHED
 from fractal_server import __VERSION__
 from fractal_server.app.db import get_sync_db
@@ -20,15 +19,10 @@ from fractal_server.app.runner.exceptions import JobExecutionError
 from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
-from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
-from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
 from fractal_server.app.runner.v2.db_tools import (
     bulk_update_status_of_history_unit,
 )
-from fractal_server.app.runner.v2.db_tools import (
-    update_logfile_of_history_unit,
-)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.config import get_settings
@@ -84,9 +78,18 @@ class BaseSlurmRunner(BaseRunner):
 
         # Create job folders. Note that the local one may or may not exist
         # depending on whether it is a test or an actual run
-        if not self.root_dir_local.is_dir():
-            self._mkdir_local_folder(self.root_dir_local.as_posix())
-        self._mkdir_remote_folder(self.root_dir_remote.as_posix())
+        try:
+            if not self.root_dir_local.is_dir():
+                self._mkdir_local_folder(self.root_dir_local.as_posix())
+            self._mkdir_remote_folder(self.root_dir_remote.as_posix())
+        except Exception as e:
+            error_msg = (
+                f"Could not mkdir {self.root_dir_local.as_posix()} or "
+                f"{self.root_dir_remote.as_posix()}. "
+                f"Original error: {str(e)}."
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
 
         self.shutdown_file = self.root_dir_local / SHUTDOWN_FILENAME
         self.jobs = {}
@@ -178,7 +181,7 @@ class BaseSlurmRunner(BaseRunner):
             fractal_server=__VERSION__,
         )
         for task in slurm_job.tasks:
-            # Wrinte input pickle
+            # Write input pickle
             _args = []
             _kwargs = dict(
                 parameters=task.parameters,
@@ -485,21 +488,14 @@ class BaseSlurmRunner(BaseRunner):
         self._mkdir_remote_folder(folder=workdir_remote.as_posix())
         logger.info("[submit] Create local/remote folders - END")
 
-        # Add prefix to task_files object
-        task_files.prefix = SUBMIT_PREFIX
-        update_logfile_of_history_unit(
-            history_unit_id=history_unit_id,
-            logfile=task_files.log_file_local,
-        )
-
         # Submission phase
         slurm_job = SlurmJob(
-            prefix=SUBMIT_PREFIX,
+            prefix=task_files.prefix,
             workdir_local=workdir_local,
             workdir_remote=workdir_remote,
             tasks=[
                 SlurmTask(
-                    prefix=SUBMIT_PREFIX,
+                    prefix=task_files.prefix,
                     index=0,
                     component=task_files.component,
                     parameters=parameters,
@@ -575,6 +571,11 @@ class BaseSlurmRunner(BaseRunner):
         task_type: Literal["parallel", "compound", "converter_compound"],
         config: SlurmConfig,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
+        """
+        Note: `list_parameters`, `list_task_files` and `history_unit_ids`
+        have the same size. For parallel tasks, this is also the number of
+        input images, while for compound tasks these can differ.
+        """
 
         if len(self.jobs) > 0:
             raise RuntimeError(
@@ -599,11 +600,7 @@ class BaseSlurmRunner(BaseRunner):
             list_parameters=list_parameters,
             task_type=task_type,
             list_task_files=list_task_files,
-        )
-        self.validate_multisubmit_history_unit_ids(
             history_unit_ids=history_unit_ids,
-            task_type=task_type,
-            list_parameters=list_parameters,
         )
 
         logger.info(f"[multisubmit] START, {len(list_parameters)=}")
@@ -624,46 +621,28 @@ class BaseSlurmRunner(BaseRunner):
 
         tot_tasks = len(list_parameters)
 
-        # Set/validate parameters for task batching
-        tasks_per_job, parallel_tasks_per_job = heuristics(
-            # Number of parallel components (always known)
-            tot_tasks=tot_tasks,
-            # Optional WorkflowTask attributes:
-            tasks_per_job=config.tasks_per_job,
-            parallel_tasks_per_job=config.parallel_tasks_per_job,  # noqa
-            # Task requirements (multiple possible sources):
-            cpus_per_task=config.cpus_per_task,
-            mem_per_task=config.mem_per_task_MB,
-            # Fractal configuration variables (soft/hard limits):
-            target_cpus_per_job=config.target_cpus_per_job,
-            target_mem_per_job=config.target_mem_per_job,
-            target_num_jobs=config.target_num_jobs,
-            max_cpus_per_job=config.max_cpus_per_job,
-            max_mem_per_job=config.max_mem_per_job,
-            max_num_jobs=config.max_num_jobs,
-        )
-        config.parallel_tasks_per_job = parallel_tasks_per_job
-        config.tasks_per_job = tasks_per_job
+        # NOTE: chunking has already taken place in `get_slurm_config`,
+        # so that `config.tasks_per_job` is now set.
 
         # Divide arguments in batches of `tasks_per_job` tasks each
         args_batches = []
-        batch_size = tasks_per_job
+        batch_size = config.tasks_per_job
         for ind_chunk in range(0, tot_tasks, batch_size):
             args_batches.append(
                 list_parameters[ind_chunk : ind_chunk + batch_size]  # noqa
             )
-        if len(args_batches) != math.ceil(tot_tasks / tasks_per_job):
+        if len(args_batches) != math.ceil(tot_tasks / config.tasks_per_job):
             raise RuntimeError("Something wrong here while batching tasks")
 
         # Part 1/3: Iterate over chunks, prepare SlurmJob objects
         logger.info("[multisubmit] Prepare `SlurmJob`s.")
         jobs_to_submit = []
         for ind_batch, chunk in enumerate(args_batches):
-            prefix = f"{MULTISUBMIT_PREFIX}-{ind_batch:06d}"
+            # Read prefix based on the first task of this batch
+            prefix = list_task_files[ind_batch * batch_size].prefix
             tasks = []
             for ind_chunk, parameters in enumerate(chunk):
                 index = (ind_batch * batch_size) + ind_chunk
-                list_task_files[index].prefix = prefix
                 tasks.append(
                     SlurmTask(
                         prefix=prefix,
@@ -676,7 +655,6 @@ class BaseSlurmRunner(BaseRunner):
                         task_files=list_task_files[index],
                     ),
                 )
-
             jobs_to_submit.append(
                 SlurmJob(
                     prefix=prefix,
@@ -697,21 +675,6 @@ class BaseSlurmRunner(BaseRunner):
                 slurm_config=config,
             )
 
-        if task_type == "parallel":
-            # FIXME: replace loop with a `bulk_update_history_unit` function
-            for ind, task_files in enumerate(list_task_files):
-                update_logfile_of_history_unit(
-                    history_unit_id=history_unit_ids[ind],
-                    logfile=task_files.log_file_local,
-                )
-        else:
-            logger.debug(
-                f"Unclear what logfile to associate to {task_type=} "
-                "within multisubmit (see issue #2382)."
-            )
-            # FIXME: Improve definition for compound tasks
-            pass
-
         logger.info(f"END submission phase, {self.job_ids=}")
 
         # FIXME: replace this sleep with a more precise check

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -2,6 +2,7 @@ from pathlib import Path
 from typing import Literal
 from typing import Optional
 
+from ._batching import heuristics
 from ._slurm_config import _parse_mem_value
 from ._slurm_config import load_slurm_config_file
 from ._slurm_config import logger
@@ -10,7 +11,7 @@ from ._slurm_config import SlurmConfigError
 from fractal_server.app.models.v2 import WorkflowTaskV2
 
 
-def get_slurm_config(
+def get_slurm_config_internal(
     wftask: WorkflowTaskV2,
     which_type: Literal["non_parallel", "parallel"],
     config_path: Optional[Path] = None,
@@ -162,3 +163,39 @@ def get_slurm_config(
     slurm_config = SlurmConfig(**slurm_dict)
 
     return slurm_config
+
+
+def get_slurm_config(
+    wftask: WorkflowTaskV2,
+    which_type: Literal["non_parallel", "parallel"],
+    config_path: Optional[Path] = None,
+    tot_tasks: int = 1,
+) -> SlurmConfig:
+    config = get_slurm_config_internal(
+        wftask,
+        which_type,
+        config_path,
+    )
+
+    # Set/validate parameters for task batching
+    tasks_per_job, parallel_tasks_per_job = heuristics(
+        # Number of parallel components (always known)
+        tot_tasks=tot_tasks,
+        # Optional WorkflowTask attributes:
+        tasks_per_job=config.tasks_per_job,
+        parallel_tasks_per_job=config.parallel_tasks_per_job,  # noqa
+        # Task requirements (multiple possible sources):
+        cpus_per_task=config.cpus_per_task,
+        mem_per_task=config.mem_per_task_MB,
+        # Fractal configuration variables (soft/hard limits):
+        target_cpus_per_job=config.target_cpus_per_job,
+        target_mem_per_job=config.target_mem_per_job,
+        target_num_jobs=config.target_num_jobs,
+        max_cpus_per_job=config.max_cpus_per_job,
+        max_mem_per_job=config.max_mem_per_job,
+        max_num_jobs=config.max_num_jobs,
+    )
+    config.parallel_tasks_per_job = parallel_tasks_per_job
+    config.tasks_per_job = tasks_per_job
+
+    return config

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/task_files.py

@@ -4,6 +4,7 @@ from typing import Union
 
 from pydantic import BaseModel
 
+from fractal_server.app.runner.components import _index_to_component
 from fractal_server.string_tools import sanitize_string
 
 SUBMIT_PREFIX = "non_par"
@@ -140,3 +141,31 @@ class TaskFiles(BaseModel):
             metadiff_file_remote=self.metadiff_file_remote,
             log_file_remote=self.log_file_remote,
         )
+
+
+def enrich_task_files_multisubmit(
+    *,
+    tot_tasks: int,
+    batch_size: int,
+    base_task_files: TaskFiles,
+) -> list[TaskFiles]:
+    """
+    Expand `TaskFiles` objects with `component` and `prefix`.
+    """
+
+    new_list_task_files: list[TaskFiles] = []
+    for absolute_index in range(tot_tasks):
+        ind_batch = absolute_index // batch_size
+        new_list_task_files.append(
+            TaskFiles(
+                **base_task_files.model_dump(
+                    exclude={
+                        "component",
+                        "prefix",
+                    }
+                ),
+                prefix=f"{MULTISUBMIT_PREFIX}-{ind_batch:06d}",
+                component=_index_to_component(absolute_index),
+            )
+        )
+    return new_list_task_files

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/v2/db_tools.py

@@ -4,7 +4,6 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 from sqlmodel import update
 
-from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
@@ -59,20 +58,6 @@ def bulk_update_status_of_history_unit(
         db_sync.commit()
 
 
-def update_logfile_of_history_unit(
-    *,
-    history_unit_id: int,
-    logfile: str,
-) -> None:
-    with next(get_sync_db()) as db_sync:
-        unit = db_sync.get(HistoryUnit, history_unit_id)
-        if unit is None:
-            raise ValueError(f"HistoryUnit {history_unit_id} not found.")
-        unit.logfile = logfile
-        db_sync.merge(unit)
-        db_sync.commit()
-
-
 def bulk_upsert_image_cache_fast(
     *,
     list_upsert_objects: list[dict[str, Any]],

{fractal_server-2.14.0a24 → fractal_server-2.14.0a25}/fractal_server/app/runner/v2/runner.py

@@ -128,10 +128,6 @@ def execute_tasks_v2(
             db.commit()
             db.refresh(history_run)
             history_run_id = history_run.id
-            logger.debug(
-                f"Created {history_run_id=}, for "
-                f"{wftask.id=} and {dataset.id=}"
-            )
 
         # TASK EXECUTION (V2)
         if task.type in ["non_parallel", "converter_non_parallel"]:
@@ -205,9 +201,9 @@ def execute_tasks_v2(
         # Update image list
         num_new_images = 0
         current_task_output.check_zarr_urls_are_unique()
-        # FIXME: Introduce for loop over task outputs, and processe them
-        # sequentially
-        # each failure should lead to an update of the specific image status
+        # NOTE: In principle we could make the task-output processing more
+        # granular, and also associate output-processing failures to history
+        # status.
         for image_obj in current_task_output.image_list_updates:
             image = image_obj.model_dump()
             # Edit existing image
@@ -346,7 +342,7 @@ def execute_tasks_v2(
             )
 
             db.commit()
-            db.close()  # FIXME: why is this needed?
+            db.close()  # NOTE: this is needed, but the reason is unclear
 
             # Create accounting record
             record = AccountingRecord(