fractal-server 2.14.0a14__py3-none-any.whl → 2.14.0a16__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a14"
+__VERSION__ = "2.14.0a16"

fractal_server/app/routes/api/v2/_aux_functions_history.py

@@ -20,6 +20,10 @@ from fractal_server.app.routes.api.v2._aux_functions import (
 from fractal_server.app.routes.api.v2._aux_functions import (
     _get_workflowtask_or_404,
 )
+from fractal_server.logger import set_logger
+
+
+logger = set_logger(__name__)
 
 
 async def get_history_unit_or_404(
@@ -67,6 +71,10 @@ def read_log_file(
     dataset_id: int,
 ):
     if logfile is None or not Path(logfile).exists():
+        logger.debug(
+            f"Logs for task '{wftask.task.name}' in dataset "
+            f"{dataset_id} are not available ({logfile=})."
+        )
         return (
             f"Logs for task '{wftask.task.name}' in dataset "
             f"{dataset_id} are not available."

fractal_server/app/routes/api/v2/history.py

@@ -84,11 +84,10 @@ async def get_workflow_tasks_statuses(
             .limit(1)
         )
         latest_history_run = res.scalar_one_or_none()
-        logger.debug(  # FIXME: remove
-            f"Given {dataset_id=} and {wftask.id}, "
-            f"found {latest_history_run=}."
-        )
         if latest_history_run is None:
+            logger.debug(
+                f"No HistoryRun found for {dataset_id=} and {wftask.id=}."
+            )
             response[wftask.id] = None
             continue
         response[wftask.id] = dict(

fractal_server/app/runner/executors/local/runner.py

@@ -11,6 +11,9 @@ from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -69,7 +72,12 @@ class LocalRunner(BaseRunner):
         workdir_local = task_files.wftask_subfolder_local
         workdir_local.mkdir()
 
+        # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # SUBMISSION PHASE
         future = self.executor.submit(
@@ -155,17 +163,35 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                current_task_files = list_task_files[positional_index]
-                current_task_files.prefix = (
-                    f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
-                )
+                list_task_files[
+                    positional_index
+                ].prefix = f"{MULTISUBMIT_PREFIX}-{positional_index:06d}"
                 future = self.executor.submit(
                     func,
                     parameters=kwargs,
-                    remote_files=current_task_files.remote_files_dict,
+                    remote_files=list_task_files[
+                        positional_index
+                    ].remote_files_dict,
                 )
                 active_futures[positional_index] = future
 
+                if task_type == "parallel":
+                    # FIXME: replace loop with a `bulk_update_history_unit`
+                    # function
+                    update_logfile_of_history_unit(
+                        history_unit_id=history_unit_ids[positional_index],
+                        logfile=list_task_files[
+                            positional_index
+                        ].log_file_local,
+                    )
+                else:
+                    logger.debug(
+                        f"Unclear what logfile to associate to {task_type=} "
+                        "within multisubmit (see issue #2382)."
+                    )
+                    # FIXME: Improve definition for compound tasks
+                    pass
+
             while active_futures:
                 # FIXME: add shutdown detection
                 # if file exists: cancel all futures, and raise

fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -23,6 +23,9 @@ from fractal_server.app.runner.filenames import SHUTDOWN_FILENAME
 from fractal_server.app.runner.task_files import MULTISUBMIT_PREFIX
 from fractal_server.app.runner.task_files import SUBMIT_PREFIX
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    update_logfile_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.config import get_settings
@@ -225,9 +228,9 @@ class BaseSlurmRunner(BaseRunner):
         logger.info(script_lines)
 
         # Always print output of `uname -n` and `pwd`
-        script_lines.append(
-            '"Hostname: `uname -n`; current directory: `pwd`"\n'
-        )
+        script_lines.append("Hostname: $(uname -n)\n")
+        script_lines.append("Current directory : $(pwd)\n")
+        script_lines.append('Start time: $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
 
         # Complete script preamble
         script_lines.append("\n")
@@ -242,6 +245,7 @@ class BaseSlurmRunner(BaseRunner):
             )
         script_lines.append("wait\n")
         script = "\n".join(script_lines)
+        script_lines.append('End time:   $(date + "%Y-%m-%dT%H:%M:%S%z")\n')
 
         # Write submission script
         with open(slurm_job.slurm_submission_script_local, "w") as f:
@@ -400,6 +404,10 @@ class BaseSlurmRunner(BaseRunner):
 
         # Add prefix to task_files object
         task_files.prefix = SUBMIT_PREFIX
+        update_logfile_of_history_unit(
+            history_unit_id=history_unit_id,
+            logfile=task_files.log_file_local,
+        )
 
         # Submission phase
         slurm_job = SlurmJob(
@@ -549,18 +557,17 @@ class BaseSlurmRunner(BaseRunner):
             tasks = []
             for ind_chunk, parameters in enumerate(chunk):
                 index = (ind_batch * batch_size) + ind_chunk
-                current_task_files = list_task_files[index]
-                current_task_files.prefix = prefix
+                list_task_files[index].prefix = prefix
                 tasks.append(
                     SlurmTask(
                         prefix=prefix,
                         index=index,
-                        component=current_task_files.component,
+                        component=list_task_files[index].component,
                         workdir_local=workdir_local,
                         workdir_remote=workdir_remote,
                         parameters=parameters,
                         zarr_url=parameters["zarr_url"],
-                        task_files=current_task_files,
+                        task_files=list_task_files[index],
                     ),
                 )
 
@@ -575,6 +582,21 @@ class BaseSlurmRunner(BaseRunner):
                 slurm_job=slurm_job,
                 slurm_config=config,
             )
+        if task_type == "parallel":
+            # FIXME: replace loop with a `bulk_update_history_unit` function
+            for ind, task_files in enumerate(list_task_files):
+                update_logfile_of_history_unit(
+                    history_unit_id=history_unit_ids[ind],
+                    logfile=task_files.log_file_local,
+                )
+        else:
+            logger.debug(
+                f"Unclear what logfile to associate to {task_type=} "
+                "within multisubmit (see issue #2382)."
+            )
+            # FIXME: Improve definition for compound tasks
+            pass
+
         logger.info(f"END submission phase, {self.job_ids=}")
 
         # FIXME: replace this sleep a more precise check

fractal_server/app/runner/v2/db_tools.py

@@ -3,11 +3,14 @@ from typing import Any
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session
 
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 
+_CHUNK_SIZE = 2_000
+
 
 def update_status_of_history_run(
     *,
@@ -37,7 +40,18 @@ def update_status_of_history_unit(
     db_sync.commit()
 
 
-_CHUNK_SIZE = 2_000
+def update_logfile_of_history_unit(
+    *,
+    history_unit_id: int,
+    logfile: str,
+) -> None:
+    with next(get_sync_db()) as db_sync:
+        unit = db_sync.get(HistoryUnit, history_unit_id)
+        if unit is None:
+            raise ValueError(f"HistoryUnit {history_unit_id} not found.")
+        unit.logfile = logfile
+        db_sync.merge(unit)
+        db_sync.commit()
 
 
 def bulk_upsert_image_cache_fast(

fractal_server/app/runner/v2/runner_functions.py

@@ -172,7 +172,7 @@ def run_v2_task_non_parallel(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=task_files.log_file_local,
+            logfile=None,
             zarr_urls=zarr_urls,
         )
         db.add(history_unit)
@@ -275,7 +275,7 @@ def run_v2_task_parallel(
         HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            logfile=list_task_files[ind].log_file_local,
+            logfile=None,
             zarr_urls=[image["zarr_url"]],
         )
         for ind, image in enumerate(images)
@@ -401,8 +401,7 @@ def run_v2_task_compound(
         history_unit = HistoryUnit(
             history_run_id=history_run_id,
             status=HistoryUnitStatus.SUBMITTED,
-            # FIXME: What about compute-task logs?
-            logfile=task_files_init.log_file_local,
+            logfile=None,
             zarr_urls=input_image_zarr_urls,
         )
         db.add(history_unit)

{fractal_server-2.14.0a14.dist-info → fractal_server-2.14.0a16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a14
+Version: 2.14.0a16
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.14.0a14.dist-info → fractal_server-2.14.0a16.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=Gb3phryDeQDKZ3Kk3X0VTqNP6DSo0hqX3pN7X5bNJRE,26
+fractal_server/__init__.py,sha256=ewSoBgac6DlCJh5FbTCwJqfzkPFKTK2DqqhRuiv9G4k,26
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -32,11 +32,11 @@ fractal_server/app/routes/admin/v2/task_group_lifecycle.py,sha256=0e0ZJ_k75TVHaT
 fractal_server/app/routes/api/__init__.py,sha256=B8l6PSAhR10iZqHEiyTat-_0tkeKdrCigIE6DJGP5b8,638
 fractal_server/app/routes/api/v2/__init__.py,sha256=9o9zxTU2IJrC_JQ8GUMft3niiBZ39YLvODUeraiRRdQ,2465
 fractal_server/app/routes/api/v2/_aux_functions.py,sha256=eE-TdEMI_UX3LBDUGwjG5NyUcihDVaHYlG15NlTJ9DI,12872
-fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=nWO4jBoL3MWuMMwS-6TwxlTHzgRr8Xed30RSeetLvP8,4199
+fractal_server/app/routes/api/v2/_aux_functions_history.py,sha256=ZlI6nwzB5r9AiY0C8TzJS_EQOTPKgkRYl3GpxFAu2bg,4430
 fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=qdXCb6IP8-qPEAxGZKljtjIqNzIAyRaAsQSRi5VqFHM,6773
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=uhNSs-jcS7ndIUFKiOC1yrDiViw3uvKEXi9UL04BMks,11642
 fractal_server/app/routes/api/v2/dataset.py,sha256=h5AhE0sdhQ20ZlIbEJsFnHIOUW0S1VHFpoflpBkVScs,8936
-fractal_server/app/routes/api/v2/history.py,sha256=Vph3SuMsg38Gt6KiaOXgdhXA0z01pu4eNQqy1CI69Kc,15957
+fractal_server/app/routes/api/v2/history.py,sha256=BuXVCDUCIYG5YF49D4b_CsqQrx6uFH6WB4TgkrdS2zo,15926
 fractal_server/app/routes/api/v2/images.py,sha256=BGpO94gVd8BTpCN6Mun2RXmjrPmfkIp73m8RN7uiGW4,8361
 fractal_server/app/routes/api/v2/job.py,sha256=MU1sHIKk_89WrD0TD44d4ufzqnywot7On_W71KjyUbQ,6500
 fractal_server/app/routes/api/v2/project.py,sha256=uAZgATiHcOvbnRX-vv1D3HoaEUvLUd7vzVmGcqOP8ZY,4602
@@ -73,13 +73,13 @@ fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 fractal_server/app/runner/executors/base_runner.py,sha256=s5aZLDPzC565FadaqFxrCLIlQzBn2D9iOpEjnBZROkk,5541
 fractal_server/app/runner/executors/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/local/get_local_config.py,sha256=wbrIYuGOvABOStrE7jNrC4ULPhtBQ5Q7Y3aKm_icomg,3508
-fractal_server/app/runner/executors/local/runner.py,sha256=o_s8DIwI0UlBQ8NozPLG9VCtvBkhlCkC7um7yAei2j4,7936
+fractal_server/app/runner/executors/local/runner.py,sha256=Kv_ZTRARAg-lAhh-4tbSE1JcwukqGzgeQk0RAMLFgGk,8963
 fractal_server/app/runner/executors/slurm_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=ZY020JZlDS5mfpgpWTChQkyHU7iLE5kx2HVd57_C6XA,8850
 fractal_server/app/runner/executors/slurm_common/_handle_exception_proxy.py,sha256=jU2N4vMafdcDPqVXwSApu4zxskCqhHmsXF3hBpOAAFA,577
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=fZaFUUXqDH0p3DndCFUpFqTqyD2tMVCuSYgYLAycpVw,15897
-fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=3IZ31YxRKt1AluSnmYs6rk3pOxi9W3bkLSgtjrW5zC8,25667
+fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=9UpCV8CyqOeLzfEiJRxm4ODeJRDqQjE2QFx8Oi9ZjOA,26629
 fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=-fAX1DZMB5RZnyYanIJD72mWOJAPkh21jd4loDXKJw4,5994
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=iXLu4d-bWzn7qmDaOjKFkcuaSHLjPESAMSLcg6c99fc,5852
 fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=YGgzTspkK9ItSMzwuYv_1tY7_1g89Qpeny5Auinxk1E,2708
@@ -99,11 +99,11 @@ fractal_server/app/runner/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 fractal_server/app/runner/v2/_local.py,sha256=DK8yagbvd6HHjcDVhUzTy0f7MURlTkQha-NM6OZKgJc,3044
 fractal_server/app/runner/v2/_slurm_ssh.py,sha256=_bytOf8z9sdrhI03D6eqg-aQPnJ7V2-qnqpcHAYizns,3278
 fractal_server/app/runner/v2/_slurm_sudo.py,sha256=DBCNxifXmMkpu71Wnk5u9-wKT7PV1WROQuY_4DYoZRI,2993
-fractal_server/app/runner/v2/db_tools.py,sha256=jj5Jlm5xPXIxpxbTyVP9QJ_8wxkzdWFU9DzgLW-8fhs,2420
+fractal_server/app/runner/v2/db_tools.py,sha256=tyLGY-g4ISZSJzk6ootEuHocdVwMg3taBL0oIivXV7M,2846
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKnvkNTanWQ-KWh_etiT08,645
 fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
 fractal_server/app/runner/v2/runner.py,sha256=SsKEZAsB8sPV8W3khTkAqaGdDwoTm_lav-fx6DdCwyA,15294
-fractal_server/app/runner/v2/runner_functions.py,sha256=GRx1oZzW1KCWq8WS57OY2XFI8cDqBk6Bv-CEeWqIaHQ,16467
+fractal_server/app/runner/v2/runner_functions.py,sha256=5cK5O2rTrCsCxMTVN3iNPRwZ_891BC9_RMo64a8ZGYw,16338
 fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=9t1CHN3EyfsGRWfG257YPY5WjQ6zuztsw_KZrpEAFPo,3703
 fractal_server/app/runner/v2/submit_workflow.py,sha256=EDUyUuIPwZHb2zm7SCRRoFsGq2cN-b5OKw6CYkZ8kWk,13048
 fractal_server/app/runner/v2/task_interface.py,sha256=IXdQTI8rXFgXv1Ez0js4CjKFf3QwO2GCHRTuwiFtiTQ,2891
@@ -208,8 +208,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=Kc_nSzdlV6KIsO0CQSPs1w70zLyENP
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.14.0a14.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.14.0a14.dist-info/METADATA,sha256=pDj7SRAY3ffZmDbLIDJoIC-rOL9_KKe5QFjurq5543Y,4563
-fractal_server-2.14.0a14.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.14.0a14.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.14.0a14.dist-info/RECORD,,
+fractal_server-2.14.0a16.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.14.0a16.dist-info/METADATA,sha256=cX3Og-LgceYk8wYfm6SlGSXr_5s-tDFI8VYJ48zPRWw,4563
+fractal_server-2.14.0a16.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.14.0a16.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.14.0a16.dist-info/RECORD,,