fractal-server 2.14.0a33__py3-none-any.whl → 2.14.0a35__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.14.0a33"
+__VERSION__ = "2.14.0a35"

fractal_server/app/runner/executors/base_runner.py

@@ -32,6 +32,7 @@ class BaseRunner(object):
         task_type: TaskTypeType,
         task_files: TaskFiles,
         config: Any,
+        user_id: int,
     ) -> tuple[Any, BaseException]:
         """
         Run a single fractal task.
@@ -44,6 +45,7 @@ class BaseRunner(object):
             task_type: Task type.
             task_files: `TaskFiles` object.
             config: Runner-specific parameters.
+            user_id:
         """
         raise NotImplementedError()
 
@@ -55,6 +57,7 @@ class BaseRunner(object):
         list_task_files: list[TaskFiles],
         task_type: TaskTypeType,
         config: Any,
+        user_id: int,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
         Run a parallel fractal task.
@@ -68,6 +71,7 @@ class BaseRunner(object):
             task_type: Task type.
             task_files: `TaskFiles` object.
             config: Runner-specific parameters.
+            user_id
         """
         raise NotImplementedError()
 

fractal_server/app/runner/executors/local/runner.py

@@ -9,6 +9,9 @@ from fractal_server.app.db import get_sync_db
 from fractal_server.app.runner.exceptions import TaskExecutionError
 from fractal_server.app.runner.executors.base_runner import BaseRunner
 from fractal_server.app.runner.task_files import TaskFiles
+from fractal_server.app.runner.v2.db_tools import (
+    bulk_update_status_of_history_unit,
+)
 from fractal_server.app.runner.v2.db_tools import update_status_of_history_unit
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.logger import set_logger
@@ -55,19 +58,35 @@ class LocalRunner(BaseRunner):
             "converter_compound",
         ],
         config: LocalBackendConfig,
+        user_id: int,
     ) -> tuple[Any, Exception]:
         logger.debug("[submit] START")
 
-        self.validate_submit_parameters(parameters, task_type=task_type)
-        workdir_local = task_files.wftask_subfolder_local
-        workdir_local.mkdir()
+        try:
+            self.validate_submit_parameters(parameters, task_type=task_type)
+            workdir_local = task_files.wftask_subfolder_local
+            workdir_local.mkdir()
 
-        # SUBMISSION PHASE
-        future = self.executor.submit(
-            func,
-            parameters=parameters,
-            remote_files=task_files.remote_files_dict,
-        )
+            # SUBMISSION PHASE
+            future = self.executor.submit(
+                func,
+                parameters=parameters,
+                remote_files=task_files.remote_files_dict,
+            )
+        except Exception as e:
+            logger.error(
+                "[submit] Unexpected exception during submission. "
+                f"Original error {str(e)}"
+            )
+            result = None
+            exception = TaskExecutionError(str(e))
+            with next(get_sync_db()) as db:
+                update_status_of_history_unit(
+                    history_unit_id=history_unit_id,
+                    status=HistoryUnitStatus.FAILED,
+                    db_sync=db,
+                )
+                return None, exception
 
         # RETRIEVAL PHASE
         with next(get_sync_db()) as db:
@@ -98,6 +117,7 @@ class LocalRunner(BaseRunner):
         list_task_files: list[TaskFiles],
         task_type: Literal["parallel", "compound", "converter_compound"],
         config: LocalBackendConfig,
+        user_id: int,
     ) -> tuple[dict[int, Any], dict[int, BaseException]]:
         """
         Note: `list_parameters`, `list_task_files` and `history_unit_ids`
@@ -105,29 +125,50 @@ class LocalRunner(BaseRunner):
         input images, while for compound tasks these can differ.
         """
 
-        self.validate_multisubmit_parameters(
-            list_parameters=list_parameters,
-            task_type=task_type,
-            list_task_files=list_task_files,
-            history_unit_ids=history_unit_ids,
-        )
-
         logger.debug(f"[multisubmit] START, {len(list_parameters)=}")
+        results: dict[int, Any] = {}
+        exceptions: dict[int, BaseException] = {}
 
-        workdir_local = list_task_files[0].wftask_subfolder_local
-        if task_type == "parallel":
-            workdir_local.mkdir()
-
-        # Set `n_elements` and `parallel_tasks_per_job`
-        n_elements = len(list_parameters)
-        parallel_tasks_per_job = config.parallel_tasks_per_job
-        if parallel_tasks_per_job is None:
-            parallel_tasks_per_job = n_elements
+        try:
+
+            self.validate_multisubmit_parameters(
+                list_parameters=list_parameters,
+                task_type=task_type,
+                list_task_files=list_task_files,
+                history_unit_ids=history_unit_ids,
+            )
+
+            workdir_local = list_task_files[0].wftask_subfolder_local
+            if task_type == "parallel":
+                workdir_local.mkdir()
+
+            # Set `n_elements` and `parallel_tasks_per_job`
+            n_elements = len(list_parameters)
+            parallel_tasks_per_job = config.parallel_tasks_per_job
+            if parallel_tasks_per_job is None:
+                parallel_tasks_per_job = n_elements
+
+        except Exception as e:
+            logger.error(
+                "[multisubmit] Unexpected exception during preliminary phase. "
+                f"Original error {str(e)}"
+            )
+            exception = TaskExecutionError(str(e))
+            exceptions = {
+                ind: exception for ind in range(len(list_parameters))
+            }
+            if task_type == "parallel":
+                with next(get_sync_db()) as db:
+                    bulk_update_status_of_history_unit(
+                        history_unit_ids=history_unit_ids,
+                        status=HistoryUnitStatus.FAILED,
+                        db_sync=db,
+                    )
+            return results, exceptions
 
         # Execute tasks, in chunks of size `parallel_tasks_per_job`
-        results: dict[int, Any] = {}
-        exceptions: dict[int, BaseException] = {}
         for ind_chunk in range(0, n_elements, parallel_tasks_per_job):
+
             list_parameters_chunk = list_parameters[
                 ind_chunk : ind_chunk + parallel_tasks_per_job
             ]
@@ -135,15 +176,31 @@ class LocalRunner(BaseRunner):
             active_futures: dict[int, Future] = {}
             for ind_within_chunk, kwargs in enumerate(list_parameters_chunk):
                 positional_index = ind_chunk + ind_within_chunk
-                future = self.executor.submit(
-                    func,
-                    parameters=kwargs,
-                    remote_files=list_task_files[
+                try:
+                    future = self.executor.submit(
+                        func,
+                        parameters=kwargs,
+                        remote_files=list_task_files[
+                            positional_index
+                        ].remote_files_dict,
+                    )
+                    active_futures[positional_index] = future
+                except Exception as e:
+                    logger.error(
+                        "[multisubmit] Unexpected exception during submission."
+                        f" Original error {str(e)}"
+                    )
+                    current_history_unit_id = history_unit_ids[
                         positional_index
-                    ].remote_files_dict,
-                )
-                active_futures[positional_index] = future
-
+                    ]
+                    exceptions[positional_index] = TaskExecutionError(str(e))
+                    if task_type == "parallel":
+                        with next(get_sync_db()) as db:
+                            update_status_of_history_unit(
+                                history_unit_id=current_history_unit_id,
+                                status=HistoryUnitStatus.FAILED,
+                                db_sync=db,
+                            )
             while active_futures:
                 finished_futures = [
                     index_and_future
@@ -171,6 +228,11 @@ class LocalRunner(BaseRunner):
                                 )
 
                         except Exception as e:
+                            logger.debug(
+                                "Multisubmit failed in retrieval "
+                                "phase with the following error "
+                                f"{str(e)}"
+                            )
                             exceptions[positional_index] = TaskExecutionError(
                                 str(e)
                             )