fractal-server 2.14.0a14__tar.gz → 2.14.0a16__tar.gz

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a14
+Version: 2.14.0a16
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a16/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a16"

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -20,6 +20,10 @@ from fractal_server.app.routes.api.v2._aux_functions import (
 from fractal_server.app.routes.api.v2._aux_functions import (
     _get_workflowtask_or_404,
 )
+from fractal_server.logger import set_logger
+
+
+logger = set_logger(__name__)
 
 
 async def get_history_unit_or_404(
@@ -67,6 +71,10 @@ def read_log_file(
     dataset_id: int,
 ):
     if logfile is None or not Path(logfile).exists():
+        logger.debug(
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available ({logfile=})."
+        )
         return (
             f"Logs for task '{wftask.task.name}' in dataset "
             f"{dataset_id} are not available."

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/routes/api/v2/history.py

@@ -84,11 +84,10 @@ async def get_workflow_tasks_statuses(
             .limit(1)
         )
         latest_history_run = res.scalar_one_or_none()
-        logger.debug(  # FIXME: remove
-            f"Given {dataset_id=} and {wftask.id}, "
-            f"found {latest_history_run=}."
-        )
         if latest_history_run is None:
+            logger.debug(
+                f"No HistoryRun found for {dataset_id=} and {wftask.id=}."
+            )
             response[wftask.id] = None
             continue
         response[wftask.id] = dict(

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/runner/executors/local/runner.py

@@ -11,6 +11,9 @@ from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -69,7 +72,12 @@ class LocalRunner(BaseRunner):
         workdir_local = task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
+        # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # SUBMISSION PHASE
         future = self.executor.submit(
@@ -155,17 +163,35 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                current_task_files = list_task_files[positional_index]
-                current_task_files.prefix = (
-                    f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
-                )
+                list_task_files[
+                    positional_index
+                ].prefix = f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
                 future = self.executor.submit(
                     func,
                     parameters=kwargs,
-                    remote_files=current_task_files.remote_files_dict,
+                    remote_files=list_task_files[
+                        positional_index
+                    ].remote_files_dict,
                 )
                 active_futures[positional_index] = future
 
+                if task_type == "parallel":
+                    # FIXME: replace loop with a `bulk_update_history_unit`
+                    # function
+                    update_logfile_of_history_unit(
+                        history_unit_id=history_unit_ids[positional_index],
+                        logfile=list_task_files[
+                            positional_index
+                        ].log_file_local,
+                    )
+                else:
+                    logger.debug(
+                        f"Unclear what logfile to associate to {task_type=} "
+                        "within multisubmit (see issue #2382)."
+                    )
+                    # FIXME: Improve definition for compound tasks
+                    pass
+
             while active_futures:
                 # FIXME: add shutdown detection
                 # if file exists: cancel all futures, and raise

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,6 +23,9 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.config import get_settings
@@ -225,9 +228,9 @@ class BaseSlurmRunner(BaseRunner):
         logger.info(script_lines)
 
         # Always print output of `uname -n` and `pwd`
-        script_lines.append(
-            '"Hostname: `uname -n`; current directory: `pwd`"\n'
-        )
+        script_lines.append("Hostname: $(uname -n)\n")
+        script_lines.append("Current directory : $(pwd)\n")
+        script_lines.append('Start time: $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
 
         # Complete script preamble
         script_lines.append("\n")
@@ -242,6 +245,7 @@ class BaseSlurmRunner(BaseRunner):
             )
         script_lines.append("wait\n")
         script = "\n".join(script_lines)
+        script_lines.append('End time:   $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
 
         # Write submission script
         with open(slurm_job.slurm_submission_script_local, "w") as f:
@@ -400,6 +404,10 @@ class BaseSlurmRunner(BaseRunner):
 
         # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # Submission phase
         slurm_job = SlurmJob(
@@ -549,18 +557,17 @@ class BaseSlurmRunner(BaseRunner):
             tasks = []
             for ind_chunk, parameters in enumerate(chunk):
                 index = (ind_batch * batch_size) + ind_chunk
-                current_task_files = list_task_files[index]
-                current_task_files.prefix = prefix
+                list_task_files[index].prefix = prefix
                 tasks.append(
                     SlurmTask(
                         prefix=prefix,
                         index=index,
-                        component=current_task_files.component,
+                        component=list_task_files[index].component,
                         workdir_local=workdir_local,
                         workdir_remote=workdir_remote,
                         parameters=parameters,
                         zarr_url=parameters["zarr_url"],
-                        task_files=current_task_files,
+                        task_files=list_task_files[index],
                     ),
                 )
 
@@ -575,6 +582,21 @@ class BaseSlurmRunner(BaseRunner):
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
+        if task_type == "parallel":
+            # FIXME: replace loop with a `bulk_update_history_unit` function
+            for ind, task_files in enumerate(list_task_files):
+                update_logfile_of_history_unit(
+                    history_unit_id=history_unit_ids[ind],
+                    logfile=task_files.log_file_local,
+                )
+        else:
+            logger.debug(
+                f"Unclear what logfile to associate to {task_type=} "
+                "within multisubmit (see issue #2382)."
+            )
+            # FIXME: Improve definition for compound tasks
+            pass
+
         logger.info(f"END submission phase, {self.job_ids=}")
 
         # FIXME: replace this sleep a more precise check

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/runner/v2/db_tools.py

@@ -3,11 +3,14 @@ from typing import Any
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 
+_CHUNK_SIZE = 2_000
+
 
 def update_status_of_history_run(
     *,
@@ -37,7 +40,18 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
-_CHUNK_SIZE = 2_000
+def update_logfile_of_history_unit(
+    *,
+    history_unit_id: int,
+    logfile: str,
+) -> None:
+    with next(get_sync_db()) as db_sync:
+        unit = db_sync.get(HistoryUnit, history_unit_id)
+        if unit is None:
+            raise ValueError(f"HistoryUnit {history_unit_id} not found.")
+        unit.logfile = logfile
+        db_sync.merge(unit)
+        db_sync.commit()
 
 
 def bulk_upsert_image_cache_fast(

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/fractal_server/app/runner/v2/runner_functions.py

@@ -172,7 +172,7 @@ def run_v2_task_non_parallel(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=task_files.log_file_local,
+            logfile=None,
             zarr_urls=zarr_urls,
         )
         db.add(history_unit)
@@ -275,7 +275,7 @@ def run_v2_task_parallel(
         HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=list_task_files[ind].log_file_local,
+            logfile=None,
             zarr_urls=[image["zarr_url"]],
         )
         for ind, image in enumerate(images)
@@ -401,8 +401,7 @@ def run_v2_task_compound(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            # FIXME: What about compute-task logs?
-            logfile=task_files_init.log_file_local,
+            logfile=None,
             zarr_urls=input_image_zarr_urls,
         )
         db.add(history_unit)

{fractal_server-2.14.0a14 → fractal_server-2.14.0a16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a14"
+version = "2.14.0a16"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a14"
+current_version = "2.14.0a16"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a14/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a14"