fractal-server 2.14.0a27__tar.gz → 2.14.0a29__tar.gz

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.0a27
+Version: 2.14.0a29
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.0a29/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.0a29"

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/models/v2/history.py

@@ -27,6 +27,7 @@ class HistoryRun(SQLModel, table=True):
         default=None,
         ondelete="SET NULL",
     )
+    job_id: int = Field(foreign_key="jobv2.id")
 
     workflowtask_dump: dict[str, Any] = Field(
         sa_column=Column(JSONB, nullable=False),

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/routes/api/v2/history.py

@@ -142,6 +142,11 @@ async def get_workflow_tasks_statuses(
                 value["num_available_images"] = None
         new_response[key] = value
 
+    for wftask in workflow.task_list:
+        logger.debug(
+            f"({dataset_id=}, {wftask.id=}): {new_response[wftask.id]}"
+        )
+
     return JSONResponse(content=new_response, status_code=200)
 
 

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/routes/api/v2/verify_image_types.py

@@ -42,7 +42,7 @@ async def verify_unique_types(
             type_filters=query.type_filters,
         )
 
-    # Get all available types (#FIXME use aux function)
+    # NOTE: see issue 2486
     available_types = set(
         _type for _img in filtered_images for _type in _img["types"].keys()
     )

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/local/runner.py

@@ -25,7 +25,6 @@ class LocalRunner(BaseRunner):
         self,
         root_dir_local: Path,
     ):
-
         self.root_dir_local = root_dir_local
         self.root_dir_local.mkdir(parents=True, exist_ok=True)
         self.executor = ThreadPoolExecutor()
@@ -182,10 +181,6 @@ class LocalRunner(BaseRunner):
                                     db_sync=db,
                                 )
 
-                            # FIXME: what should happen here? Option 1: stop
-                            # all existing tasks and shutdown runner (for the
-                            # compound-task case)
-
-        logger.debug(f"[multisubmit] END, {results=}, {exceptions=}")
+        logger.debug(f"[multisubmit] END, {len(results)=}, {len(exceptions)=}")
 
         return results, exceptions

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/slurm_common/_slurm_config.py

@@ -368,9 +368,7 @@ class SlurmConfig(BaseModel):
             if value is not None:
                 # Handle the `time` parameter
                 if key == "time" and self.parallel_tasks_per_job > 1:
-                    # FIXME SSH: time setting must be handled better. Right now
-                    # we simply propagate `time`, but this is not enough when
-                    # several `srun` are combined in a single script.
+                    # NOTE: see issue #1632
                     logger.warning(
                         f"`time` SLURM parameter is set to {self.time}, "
                         "but this does not take into account the number of "

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py

@@ -34,7 +34,7 @@ SHUTDOWN_EXCEPTION = JobExecutionError(SHUTDOWN_ERROR_MESSAGE)
 
 logger = set_logger(__name__)
 
-# FIXME: Transform several logger.info into logger.debug.
+# NOTE: see issue 2481.
 
 
 class BaseSlurmRunner(BaseRunner):
@@ -107,8 +107,7 @@ class BaseSlurmRunner(BaseRunner):
         raise NotImplementedError("Implement in child class.")
 
     def run_squeue(self, job_ids: list[str]) -> tuple[bool, str]:
-
-        # FIXME: review different cases (exception vs no job found)
+        # NOTE: see issue 2482
 
         if len(job_ids) == 0:
             return (False, "")
@@ -457,7 +456,6 @@ class BaseSlurmRunner(BaseRunner):
             "converter_compound",
         ],
     ) -> tuple[Any, Exception]:
-
         logger.info("[submit] START")
 
         workdir_local = task_files.wftask_subfolder_local
@@ -514,7 +512,7 @@ class BaseSlurmRunner(BaseRunner):
         )
         logger.info(f"[submit] END submission phase, {self.job_ids=}")
 
-        # FIXME: replace this sleep with a more precise check
+        # NOTE: see issue 2444
         settings = Inject(get_settings)
         sleep_time = settings.FRACTAL_SLURM_INTERVAL_BEFORE_RETRIEVAL
         logger.warning(f"[submit] Now sleep {sleep_time} seconds.")
@@ -524,7 +522,6 @@ class BaseSlurmRunner(BaseRunner):
         logger.info("[submit] START retrieval phase")
         scancelled_job_ids = []
         while len(self.jobs) > 0:
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[submit] {finished_job_ids=}")
@@ -664,9 +661,7 @@ class BaseSlurmRunner(BaseRunner):
                 )
             )
 
-        # FIXME: split parts 2 and 3
-        # Part 2/3. Transfer all relevant input files (for SSH)
-        # Part 3/3. Run all `sbatch`es and update `self.jobs`
+        # NOTE: see issue 2431
         logger.info("[multisubmit] Transfer files and submit jobs.")
         for slurm_job in jobs_to_submit:
             self._submit_single_sbatch(
@@ -677,19 +672,15 @@ class BaseSlurmRunner(BaseRunner):
 
         logger.info(f"END submission phase, {self.job_ids=}")
 
-        # FIXME: replace this sleep with a more precise check
         settings = Inject(get_settings)
         sleep_time = settings.FRACTAL_SLURM_INTERVAL_BEFORE_RETRIEVAL
         logger.warning(f"[submit] Now sleep {sleep_time} seconds.")
         time.sleep(sleep_time)
 
-        # FIXME: Could we merge the submit/multisubmit retrieval phases?
-
         # Retrieval phase
         logger.info("[multisubmit] START retrieval phase")
         scancelled_job_ids = []
         while len(self.jobs) > 0:
-
             # Look for finished jobs
             finished_job_ids = self._get_finished_jobs(job_ids=self.job_ids)
             logger.debug(f"[multisubmit] {finished_job_ids=}")

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/slurm_common/remote.py

@@ -59,7 +59,7 @@ def _check_versions_mismatch(
     if worker_python_version != server_python_version:
         # FIXME: turn this into an error, after fixing a broader CI issue, see
         # https://github.com/fractal-analytics-platform/fractal-server/issues/375
-        logging.critical(
+        logging.warning(
             f"{server_python_version=} but {worker_python_version=}. "
             "cloudpickle is not guaranteed to correctly load "
             "pickle files created with different python versions. "

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/slurm_ssh/runner.py

@@ -90,7 +90,7 @@ class SlurmSSHRunner(BaseSlurmRunner):
         ).as_posix()
 
         # Create file list
-        # # FIXME can we make this more efficient with iterators?
+        # NOTE: see issue 2483
         filelist = []
         for _slurm_job in finished_slurm_jobs:
             _single_job_filelist = [
@@ -168,7 +168,6 @@ class SlurmSSHRunner(BaseSlurmRunner):
         Transfer the jobs subfolder to the remote host.
         """
         for job in jobs:
-
             # Create local archive
             tarfile_path_local = compress_folder(
                 job.workdir_local,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/executors/slurm_sudo/runner.py

@@ -133,7 +133,10 @@ class SudoSlurmRunner(BaseSlurmRunner):
                 # Write local file
                 with open(target, "wb") as f:
                     f.write(res.stdout)
-                logger.critical(f"Copied {source} into {target}")
+                logger.debug(
+                    f"[_fetch_artifacts_single_job] Copied {source} into "
+                    f"{target}"
+                )
             except RuntimeError as e:
                 logger.warning(
                     f"SKIP copy {source} into {target}. "

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/_local.py

@@ -15,6 +15,7 @@ def process_workflow(
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
+    job_id: int,
     workflow_dir_remote: Optional[Path] = None,
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
@@ -75,6 +76,7 @@ def process_workflow(
                 first_task_index : (last_task_index + 1)
             ],
             dataset=dataset,
+            job_id=job_id,
             runner=runner,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_local,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/_slurm_ssh.py

@@ -38,6 +38,7 @@ def process_workflow(
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
+    job_id: int,
     workflow_dir_remote: Optional[Path] = None,
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
@@ -87,6 +88,7 @@ def process_workflow(
                 first_task_index : (last_task_index + 1)
             ],
             dataset=dataset,
+            job_id=job_id,
             runner=runner,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/_slurm_sudo.py

@@ -33,6 +33,7 @@ def process_workflow(
     workflow: WorkflowV2,
     dataset: DatasetV2,
     workflow_dir_local: Path,
+    job_id: int,
     workflow_dir_remote: Optional[Path] = None,
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
@@ -79,6 +80,7 @@ def process_workflow(
                 first_task_index : (last_task_index + 1)
             ],
             dataset=dataset,
+            job_id=job_id,
             runner=runner,
             workflow_dir_local=workflow_dir_local,
             workflow_dir_remote=workflow_dir_remote,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/db_tools.py

@@ -8,9 +8,13 @@ from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryRun
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
+from fractal_server.logger import set_logger
+
 
 _CHUNK_SIZE = 2_000
 
+logger = set_logger(__name__)
+
 
 def update_status_of_history_run(
     *,
@@ -46,7 +50,12 @@ def bulk_update_status_of_history_unit(
     status: HistoryUnitStatus,
     db_sync: Session,
 ) -> None:
-    for ind in range(0, len(history_unit_ids), _CHUNK_SIZE):
+
+    len_history_unit_ids = len(history_unit_ids)
+    logger.debug(
+        f"[bulk_update_status_of_history_unit] {len_history_unit_ids=}."
+    )
+    for ind in range(0, len_history_unit_ids, _CHUNK_SIZE):
         db_sync.execute(
             update(HistoryUnit)
             .where(
@@ -77,7 +86,7 @@ def bulk_upsert_image_cache_fast(
     See docs at
     https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#insert-on-conflict-upsert
 
-    FIXME: we tried to replace `index_elements` with
+    NOTE: we tried to replace `index_elements` with
     `constraint="pk_historyimagecache"`, but it did not work as expected.
 
     Arguments:
@@ -85,10 +94,14 @@ def bulk_upsert_image_cache_fast(
             List of dictionaries for objects to be upsert-ed.
         db: A sync database session
     """
-    if len(list_upsert_objects) == 0:
+    len_list_upsert_objects = len(list_upsert_objects)
+
+    logger.debug(f"[bulk_upsert_image_cache_fast] {len_list_upsert_objects=}.")
+
+    if len_list_upsert_objects == 0:
         return None
 
-    for ind in range(0, len(list_upsert_objects), _CHUNK_SIZE):
+    for ind in range(0, len_list_upsert_objects, _CHUNK_SIZE):
         stmt = pg_insert(HistoryImageCache).values(
             list_upsert_objects[ind : ind + _CHUNK_SIZE]
         )

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/runner.py

@@ -42,6 +42,7 @@ def execute_tasks_v2(
     runner: BaseRunner,
     user_id: int,
     workflow_dir_local: Path,
+    job_id: int,
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
     get_runner_config: Callable[
@@ -119,6 +120,7 @@ def execute_tasks_v2(
             history_run = HistoryRun(
                 dataset_id=dataset.id,
                 workflowtask_id=wftask.id,
+                job_id=job_id,
                 workflowtask_dump=workflowtask_dump,
                 task_group_dump=task_group_dump,
                 num_available_images=num_available_images,
@@ -128,6 +130,10 @@ def execute_tasks_v2(
             db.commit()
             db.refresh(history_run)
             history_run_id = history_run.id
+            logger.debug(
+                "[execute_tasks_v2] Created `HistoryRun` with "
+                f"{history_run_id=}."
+            )
 
         # TASK EXECUTION (V2)
         if task.type in ["non_parallel", "converter_non_parallel"]:

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/runner_functions.py

@@ -50,6 +50,7 @@ class SubmissionOutcome(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     task_output: TaskOutput | None = None
     exception: BaseException | None = None
+    invalid_output: bool = False
 
 
 class InitSubmissionOutcome(BaseModel):
@@ -66,6 +67,7 @@ def _process_task_output(
     result: dict[str, Any] | None = None,
     exception: BaseException | None = None,
 ) -> SubmissionOutcome:
+    invalid_output = False
     if exception is not None:
         task_output = None
     else:
@@ -75,13 +77,13 @@ def _process_task_output(
             try:
                 task_output = _cast_and_validate_TaskOutput(result)
             except TaskOutputValidationError as e:
-                # FIXME: This should correspond to some status="failed",
-                # but it does not
                 task_output = None
                 exception = e
+                invalid_output = True
     return SubmissionOutcome(
         task_output=task_output,
         exception=exception,
+        invalid_output=invalid_output,
     )
 
 
@@ -99,8 +101,6 @@ def _process_init_task_output(
             try:
                 task_output = _cast_and_validate_InitTaskOutput(result)
             except TaskOutputValidationError as e:
-                # FIXME: This should correspond to some status="failed",
-                # but it does not
                 task_output = None
                 exception = e
     return InitSubmissionOutcome(
@@ -187,6 +187,10 @@ def run_v2_task_non_parallel(
         db.add(history_unit)
         db.commit()
         db.refresh(history_unit)
+        logger.debug(
+            "[run_v2_task_non_parallel] Created `HistoryUnit` with "
+            f"{history_run_id=}."
+        )
         history_unit_id = history_unit.id
         bulk_upsert_image_cache_fast(
             db=db,
@@ -225,6 +229,13 @@ def run_v2_task_non_parallel(
             exception=exception,
         )
     }
+    if outcome[0].invalid_output:
+        with next(get_sync_db()) as db:
+            update_status_of_history_unit(
+                history_unit_id=history_unit_id,
+                status=HistoryUnitStatus.FAILED,
+                db_sync=db,
+            )
     return outcome, num_tasks
 
 
@@ -294,6 +305,10 @@ def run_v2_task_parallel(
     with next(get_sync_db()) as db:
         db.add_all(history_units)
         db.commit()
+        logger.debug(
+            f"[run_v2_task_non_parallel] Created {len(history_units)} "
+            "`HistoryUnit`s."
+        )
 
         for history_unit in history_units:
             db.refresh(history_unit)
@@ -331,7 +346,6 @@ def run_v2_task_parallel(
     outcome = {}
     for ind in range(len(list_function_kwargs)):
         if ind not in results.keys() and ind not in exceptions.keys():
-            # FIXME: Could we avoid this branch?
             error_msg = (
                 f"Invalid branch: {ind=} is not in `results.keys()` "
                 "nor in `exceptions.keys()`."
@@ -342,7 +356,13 @@ def run_v2_task_parallel(
             result=results.get(ind, None),
             exception=exceptions.get(ind, None),
         )
-
+        if outcome[ind].invalid_output:
+            with next(get_sync_db()) as db:
+                update_status_of_history_unit(
+                    history_unit_id=history_unit_ids[ind],
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
     num_tasks = len(images)
     return outcome, num_tasks
 
@@ -407,6 +427,10 @@ def run_v2_task_compound(
         db.commit()
         db.refresh(history_unit)
         init_history_unit_id = history_unit.id
+        logger.debug(
+            "[run_v2_task_compound] Created `HistoryUnit` with "
+            f"{init_history_unit_id=}."
+        )
         # Create one `HistoryImageCache` for each input image
         bulk_upsert_image_cache_fast(
             db=db,
@@ -524,6 +548,10 @@ def run_v2_task_compound(
         db.commit()
         for history_unit in history_units:
             db.refresh(history_unit)
+        logger.debug(
+            f"[run_v2_task_compound] Created {len(history_units)} "
+            "`HistoryUnit`s."
+        )
         history_unit_ids = [history_unit.id for history_unit in history_units]
 
     results, exceptions = runner.multisubmit(
@@ -545,7 +573,7 @@ def run_v2_task_compound(
     failure = False
     for ind in range(len(list_function_kwargs)):
         if ind not in results.keys() and ind not in exceptions.keys():
-            # FIXME: Could we avoid this branch?
+            # NOTE: see issue 2484
             error_msg = (
                 f"Invalid branch: {ind=} is not in `results.keys()` "
                 "nor in `exceptions.keys()`."
@@ -556,7 +584,7 @@ def run_v2_task_compound(
             result=results.get(ind, None),
             exception=exceptions.get(ind, None),
         )
-        if compute_outcomes[ind].exception is not None:
+        if compute_outcomes[ind].invalid_output:
             failure = True
 
     # NOTE: For compound tasks, we update `HistoryUnit.status` from here,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -84,13 +84,12 @@ def run_single_task(
     logger.debug(f"Now start running {command=}")
 
     # Write arguments to args.json file
-    # FIXME: this could be done backend-side, with an additional
-    # file transfer if needed (e.g. on SSH)
+    # NOTE: see issue 2346
     with open(args_file_remote, "w") as f:
         json.dump(parameters, f, indent=2)
 
     # Assemble full command
-    # FIXME: this could be assembled backend-side
+    # NOTE: this could be assembled backend-side
     full_command = (
         f"{command} "
         f"--args-json {args_file_remote} "

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/runner/v2/submit_workflow.py

@@ -282,6 +282,7 @@ def submit_workflow(
         process_workflow(
             workflow=workflow,
             dataset=dataset,
+            job_id=job_id,
             user_id=user_id,
             workflow_dir_local=WORKFLOW_DIR_LOCAL,
             workflow_dir_remote=WORKFLOW_DIR_REMOTE,

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/app/schemas/v2/history.py

@@ -45,6 +45,7 @@ class HistoryRunRead(BaseModel):
     id: int
     dataset_id: int
     workflowtask_id: Optional[int] = None
+    job_id: int
     workflowtask_dump: dict[str, Any]
     task_group_dump: dict[str, Any]
     timestamp_started: AwareDatetime

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/fractal_server/config.py

@@ -494,7 +494,7 @@ class Settings(BaseSettings):
 
     FRACTAL_SLURM_INTERVAL_BEFORE_RETRIEVAL: int = 2
     """
-    FIXME: this is a workaround, we are still investigating.
+    NOTE: see issue 2444
     """
 
     FRACTAL_SLURM_SBATCH_SLEEP: float = 0

fractal_server-2.14.0a29/fractal_server/migrations/versions/c90a7c76e996_job_id_in_history_run.py

@@ -0,0 +1,41 @@
+"""job id in history run
+
+Revision ID: c90a7c76e996
+Revises: f37aceb45062
+Create Date: 2025-04-16 10:44:30.219309
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "c90a7c76e996"
+down_revision = "f37aceb45062"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyrun", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("job_id", sa.Integer(), nullable=False))
+        batch_op.create_foreign_key(
+            batch_op.f("fk_historyrun_job_id_jobv2"),
+            "jobv2",
+            ["job_id"],
+            ["id"],
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("historyrun", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_historyrun_job_id_jobv2"), type_="foreignkey"
+        )
+        batch_op.drop_column("job_id")
+
+    # ### end Alembic commands ###

{fractal_server-2.14.0a27 → fractal_server-2.14.0a29}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.14.0a27"
+version = "2.14.0a29"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh"]
 
 [tool.bumpver]
-current_version = "2.14.0a27"
+current_version = "2.14.0a29"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.14.0a27/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.14.0a27"