fractal-server 2.15.4__py3-none-any.whl → 2.15.6__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.15.4"
+__VERSION__ = "2.15.6"

fractal_server/app/routes/admin/v2/job.py

@@ -12,12 +12,15 @@ from sqlmodel import select
 from fractal_server.app.db import AsyncSession
 from fractal_server.app.db import get_async_db
 from fractal_server.app.models import UserOAuth
+from fractal_server.app.models.v2 import HistoryRun
+from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.models.v2 import JobV2
 from fractal_server.app.models.v2 import ProjectV2
 from fractal_server.app.routes.auth import current_active_superuser
 from fractal_server.app.routes.aux._job import _write_shutdown_file
 from fractal_server.app.routes.aux._runner import _check_shutdown_is_supported
 from fractal_server.app.runner.filenames import WORKFLOW_LOG_FILENAME
+from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import JobReadV2
 from fractal_server.app.schemas.v2 import JobStatusTypeV2
 from fractal_server.app.schemas.v2 import JobUpdateV2
@@ -148,6 +151,11 @@ async def update_job(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=f"Job {job_id} not found",
         )
+    if job.status != JobStatusTypeV2.SUBMITTED:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Job {job_id} has status {job.status=} != 'submitted'.",
+        )
 
     if job_update.status != JobStatusTypeV2.FAILED:
         raise HTTPException(
@@ -164,6 +172,25 @@ async def update_job(
         f"{job.log or ''}\nThis job was manually marked as "
         f"'{JobStatusTypeV2.FAILED}' by an admin ({timestamp.isoformat()}).",
     )
+
+    res = await db.execute(
+        select(HistoryRun)
+        .where(HistoryRun.job_id == job_id)
+        .order_by(HistoryRun.timestamp_started.desc())
+        .limit(1)
+    )
+    latest_run = res.scalar_one_or_none()
+    if latest_run is not None:
+        setattr(latest_run, "status", HistoryUnitStatus.FAILED)
+        res = await db.execute(
+            select(HistoryUnit).where(
+                HistoryUnit.history_run_id == latest_run.id
+            )
+        )
+        history_units = res.scalars().all()
+        for history_unit in history_units:
+            setattr(history_unit, "status", HistoryUnitStatus.FAILED)
+
     await db.commit()
     await db.refresh(job)
     await db.close()

fractal_server/app/runner/exceptions.py

@@ -1,6 +1,3 @@
-import os
-
-
 class TaskExecutionError(RuntimeError):
     """
     Forwards errors occurred during the execution of a task
@@ -42,86 +39,27 @@ class TaskOutputValidationError(ValueError):
 
 class JobExecutionError(RuntimeError):
     """
-    Forwards errors in the execution of a task that are due to external factors
-
-    This error wraps and forwards errors occurred during the execution of
-    tasks, but related to external factors like an error on the executor side.
-
-    This error also adds information that is useful to track down and debug the
-    failing task within a workflow.
+    JobExecutionError
 
     Attributes:
         info:
             A free field for additional information
-        cmd_file:
-            Path to the file of the command that was executed (e.g. a SLURM
-            submission script).
-        stdout_file:
-            Path to the file with the command stdout
-        stderr_file:
-            Path to the file with the command stderr
     """
 
-    cmd_file: str | None = None
-    stdout_file: str | None = None
-    stderr_file: str | None = None
     info: str | None = None
 
     def __init__(
         self,
         *args,
-        cmd_file: str | None = None,
-        stdout_file: str | None = None,
-        stderr_file: str | None = None,
         info: str | None = None,
     ):
         super().__init__(*args)
-        self.cmd_file = cmd_file
-        self.stdout_file = stdout_file
-        self.stderr_file = stderr_file
         self.info = info
 
-    def _read_file(self, filepath: str) -> str:
-        """
-        Return the content of a text file, and handle the cases where it is
-        empty or missing
-        """
-        if os.path.exists(filepath):
-            with open(filepath) as f:
-                content = f.read()
-                if content:
-                    return f"Content of {filepath}:\n{content}"
-                else:
-                    return f"File {filepath} is empty\n"
-        else:
-            return f"File {filepath} is missing\n"
-
     def assemble_error(self) -> str:
-        """
-        Read the files that are specified in attributes, and combine them in an
-        error message.
-        """
-        if self.cmd_file:
-            content = self._read_file(self.cmd_file)
-            cmd_content = f"COMMAND:\n{content}\n\n"
-        else:
-            cmd_content = ""
-        if self.stdout_file:
-            content = self._read_file(self.stdout_file)
-            out_content = f"STDOUT:\n{content}\n\n"
-        else:
-            out_content = ""
-        if self.stderr_file:
-            content = self._read_file(self.stderr_file)
-            err_content = f"STDERR:\n{content}\n\n"
-        else:
-            err_content = ""
-
-        content = f"{cmd_content}{out_content}{err_content}"
         if self.info:
-            content = f"{content}ADDITIONAL INFO:\n{self.info}\n\n"
-
-        if not content:
+            content = f"\n{self.info}\n\n"
+        else:
             content = str(self)
-        message = f"JobExecutionError\n\n{content}"
+        message = f"JobExecutionError\n{content}"
         return message

fractal_server/app/runner/executors/slurm_common/get_slurm_config.py

@@ -8,6 +8,7 @@ from ._slurm_config import logger
 from ._slurm_config import SlurmConfig
 from ._slurm_config import SlurmConfigError
 from fractal_server.app.models.v2 import WorkflowTaskV2
+from fractal_server.string_tools import interpret_as_bool
 
 
 def get_slurm_config_internal(
@@ -86,7 +87,7 @@ def get_slurm_config_internal(
     # 2. This block of definitions has lower priority than whatever comes next
     #    (i.e. from WorkflowTask.meta_parallel).
     if wftask_meta is not None:
-        needs_gpu = wftask_meta.get("needs_gpu", False)
+        needs_gpu = interpret_as_bool(wftask_meta.get("needs_gpu", False))
     else:
         needs_gpu = False
     logger.debug(f"[get_slurm_config] {needs_gpu=}")

fractal_server/app/runner/v2/submit_workflow.py

@@ -25,7 +25,6 @@ from ...models.v2 import JobV2
 from ...models.v2 import WorkflowV2
 from ...schemas.v2 import JobStatusTypeV2
 from ..exceptions import JobExecutionError
-from ..exceptions import TaskExecutionError
 from ..filenames import WORKFLOW_LOG_FILENAME
 from ._local import process_workflow as local_process_workflow
 from ._slurm_ssh import process_workflow as slurm_ssh_process_workflow
@@ -311,19 +310,6 @@ def submit_workflow(
         db_sync.merge(job)
         db_sync.commit()
 
-    except TaskExecutionError as e:
-        logger.debug(f'FAILED workflow "{workflow.name}", TaskExecutionError.')
-        logger.info(f'Workflow "{workflow.name}" failed (TaskExecutionError).')
-
-        exception_args_string = "\n".join(e.args)
-        log_msg = (
-            f"TASK ERROR: "
-            f"Task name: {e.task_name}, "
-            f"position in Workflow: {e.workflow_task_order}\n"
-            f"TRACEBACK:\n{exception_args_string}"
-        )
-        fail_job(db=db_sync, job=job, log_msg=log_msg, logger_name=logger_name)
-
     except JobExecutionError as e:
         logger.debug(f'FAILED workflow "{workflow.name}", JobExecutionError.')
         logger.info(f'Workflow "{workflow.name}" failed (JobExecutionError).')

fractal_server/string_tools.py

@@ -60,3 +60,26 @@ def validate_cmd(
             f"'{forbidden}'\n"
             f"Provided {attribute_name.lower()}: '{command}'."
         )
+
+
+def interpret_as_bool(value: bool | str) -> bool:
+    """
+    Interpret a boolean or a string representation of a boolean as a boolean
+    value.
+
+    Accepts either a boolean (`True` or `False`) or a case-insensitive string
+    representation of a boolean ("true" or "false").
+    Returns the corresponding boolean value.
+    """
+    if isinstance(value, bool):
+        return value
+    elif isinstance(value, str):
+        value_lower = value.lower()
+        if value_lower == "true":
+            return True
+        elif value_lower == "false":
+            return False
+        else:
+            raise ValueError("String must be 'true' or 'false'.")
+    else:
+        raise TypeError(f"Expected bool or str, got {type(value)}: '{value}'.")

fractal_server/tasks/v2/ssh/_utils.py

@@ -26,6 +26,7 @@ def _customize_and_run_template(
     fractal_ssh: FractalSSH,
     script_dir_remote: str,
     logger_name: str,
+    login_shell: bool = False,
 ) -> str:
     """
     Customize one of the template bash scripts, transfer it to the remote host
@@ -38,6 +39,7 @@ def _customize_and_run_template(
         prefix: Prefix for the script filename.
         fractal_ssh: FractalSSH object
         script_dir_remote: Remote scripts directory
+        login_shell: If `True`, use `bash --login` for remote script execution.
     """
     logger = get_logger(logger_name=logger_name)
     logger.debug(f"_customize_and_run_template {template_filename} - START")
@@ -67,7 +69,8 @@ def _customize_and_run_template(
     )
 
     # Execute script remotely
-    cmd = f"bash {script_path_remote}"
+    bash = "bash --login " if login_shell else "bash"
+    cmd = f"{bash} {script_path_remote}"
     logger.debug(f"Now run '{cmd}' over SSH.")
     stdout = fractal_ssh.run_command(cmd=cmd)
 

fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -204,6 +204,7 @@ def collect_ssh_pixi(
                     stdout = _customize_and_run_template(
                         template_filename="pixi_2_install.sh",
                         replacements=replacements,
+                        login_shell=True,
                         **common_args,
                     )
                     logger.debug(f"STDOUT: {stdout}")

fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -190,6 +190,7 @@ def reactivate_ssh_pixi(
                     stdout = _customize_and_run_template(
                         template_filename="pixi_2_install.sh",
                         replacements=replacements,
+                        login_shell=True,
                         **common_args,
                     )
                     logger.debug(f"STDOUT: {stdout}")

{fractal_server-2.15.4.dist-info → fractal_server-2.15.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.15.4
+Version: 2.15.6
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

{fractal_server-2.15.4.dist-info → fractal_server-2.15.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=oQyZUHcUuOnv7ZqsoVjwEhBSMPbobwbvTyb3GUD_ZT8,23
+fractal_server/__init__.py,sha256=9b_JI6UfsPyLDqzYMe_YaGYPHyDEJP2xKvdlE-7QIqo,23
 fractal_server/__main__.py,sha256=rkM8xjY1KeS3l63irB8yCrlVobR-73uDapC4wvrIlxI,6957
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,7 +23,7 @@ fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm
 fractal_server/app/routes/admin/v2/__init__.py,sha256=_5lqb6-M8-fZqE1HRMep6pAFYRUKMxrvbZOKs-RXWkw,933
 fractal_server/app/routes/admin/v2/accounting.py,sha256=-eLzBirENvdhjhrCf6bs1pTriIui19DAr_k-zuJB7Y8,3593
 fractal_server/app/routes/admin/v2/impersonate.py,sha256=gc4lshfEPFR6W2asH7aKu6hqE6chzusdhAUVV9p51eU,1131
-fractal_server/app/routes/admin/v2/job.py,sha256=fl_1hWrriCA24OPlHZhxfnM1j0C4UL5hqjcG1ww8Fac,7584
+fractal_server/app/routes/admin/v2/job.py,sha256=ysfrmsaLUwdo3_YbXOhh6j2zZqKrj5dF2s4gJf1ngtQ,8604
 fractal_server/app/routes/admin/v2/project.py,sha256=MA_LdoEuSuisSGRO43TapMuJ080y5iaUGSAUgKuuKOg,1188
 fractal_server/app/routes/admin/v2/task.py,sha256=8njjq_zcvNW-Ewxn7WfsRQbs_aV5h-7pgcRIAegzTnc,4308
 fractal_server/app/routes/admin/v2/task_group.py,sha256=JUCxKnwSNmOKyh6lpqfG5jT5BuRE_U2X6HqD5VVhWPg,7096
@@ -70,7 +70,7 @@ fractal_server/app/routes/aux/validate_user_settings.py,sha256=FLVi__8YFcm_6c_K5
 fractal_server/app/routes/pagination.py,sha256=IGy8Ll5lYr6ENYE18h3huH5s0GMX1DCs5VdCi6j1cdk,1174
 fractal_server/app/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/components.py,sha256=-Ii5l8d_V6f5DFOd-Zsr8VYmOsyqw0Hox9fEFQiuqxY,66
-fractal_server/app/runner/exceptions.py,sha256=tJxs7WCQ86kjezunFm4o_VAiUAyD70l3GiH6ht0waWA,3958
+fractal_server/app/runner/exceptions.py,sha256=lzYefT6IpA2ajcQUzm9kT9pTxRl5UGIJKyLs6v1JczA,1731
 fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/base_runner.py,sha256=7ujN6gN92X9sWAlDwkc07G3FA4Z_Zwew_uBYh2qv0uI,5978
 fractal_server/app/runner/executors/call_command_wrapper.py,sha256=1BHl-zbXoX2oGUWGAFprVZMmg5QjutPH0-VZJSIC0II,1419
@@ -82,7 +82,7 @@ fractal_server/app/runner/executors/slurm_common/_batching.py,sha256=gbHZIxt90Gj
 fractal_server/app/runner/executors/slurm_common/_job_states.py,sha256=nuV-Zba38kDrRESOVB3gaGbrSPZc4q7YGichQaeqTW0,238
 fractal_server/app/runner/executors/slurm_common/_slurm_config.py,sha256=U9BONnnwn8eDqDevwUtFSBcvIsxvNgDHirhcQGJ9t9E,15947
 fractal_server/app/runner/executors/slurm_common/base_slurm_runner.py,sha256=1Sh56lb7NERVtsBMvVs4K7nVHhMy_KDbwquPl1ub8vE,37937
-fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=jhoFHauWJm55bIC_v7pFylbK8WgcRJemGu2OjUiRbpQ,7377
+fractal_server/app/runner/executors/slurm_common/get_slurm_config.py,sha256=KhQQJrGQZLX5u8fsMcIx7FN8jUKSGEeG68yO7Ay_LXg,7454
 fractal_server/app/runner/executors/slurm_common/remote.py,sha256=LHK2Ram8X8q6jNSCxnnwKUwmSJMsyQyRem_VjH53qdw,3811
 fractal_server/app/runner/executors/slurm_common/slurm_job_task_models.py,sha256=K4SdJOKsUWzDlnkb8Ug_UmTx6nBMsTqn9_oKqwE4XDI,3520
 fractal_server/app/runner/executors/slurm_ssh/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -105,7 +105,7 @@ fractal_server/app/runner/v2/deduplicate_list.py,sha256=IVTE4abBU1bUprFTkxrTfYKn
 fractal_server/app/runner/v2/merge_outputs.py,sha256=YOTKbGOM9s-uqY4KN2onoIxuHNm-v3hr5zv6Aa1KEtA,905
 fractal_server/app/runner/v2/runner.py,sha256=IsQhSxASVI-yPBO2G2uuVvZkRIoAlSY6Ev7gcSYmDOw,18989
 fractal_server/app/runner/v2/runner_functions.py,sha256=aEwEDzI2l-QvgfJSj-M2LGvqA89nOiJYIFVryKEq_3M,18988
-fractal_server/app/runner/v2/submit_workflow.py,sha256=AMnXdozwIGlXD55ch0_SNAG-ntKBO-QRhkbInrvsShU,13140
+fractal_server/app/runner/v2/submit_workflow.py,sha256=L6WqQC3Vm-tLe_CJXHIMbYLAeulAN2no_4HZD4cbhR0,12554
 fractal_server/app/runner/v2/task_interface.py,sha256=BRSKpitGproY48JQdCbfrghbDonA-EqPP1yIopohpPo,2525
 fractal_server/app/runner/versions.py,sha256=4BW-8Et8RVgILgpFoUJLWkEnZz53pv8hv_2ucG480ns,398
 fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMoqWc3orFyI,135
@@ -186,7 +186,7 @@ fractal_server/migrations/versions/fbce16ff4e47_new_history_items.py,sha256=TDWC
 fractal_server/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/ssh/__init__.py,sha256=sVUmzxf7_DuXG1xoLQ1_00fo5NPhi2LJipSmU5EAkPs,124
 fractal_server/ssh/_fabric.py,sha256=7fCxTYqkAOaTTm67trfYdYQenOsI4EfrRQoG6x3M5kk,25188
-fractal_server/string_tools.py,sha256=qLB5u6-4QxXPiZrUeWn_cEo47axj4OXFzDd47kNTIWw,1847
+fractal_server/string_tools.py,sha256=UJFi8rhlI6QXxX5twycLjsvOQ6x4uG7L3JdxEVDhC5A,2592
 fractal_server/syringe.py,sha256=3YJeIALH-wibuJ9R5VMNYUWh7x1-MkWT0SqGcWG5MY8,2795
 fractal_server/tasks/__init__.py,sha256=kadmVUoIghl8s190_Tt-8f-WBqMi8u8oU4Pvw39NHE8,23
 fractal_server/tasks/utils.py,sha256=V7dj8o2AnoHhGSTYlqJHcRFhCIpmOrMOUhtiE_DvRVA,291
@@ -200,13 +200,13 @@ fractal_server/tasks/v2/local/deactivate_pixi.py,sha256=x9YCILEP1EjCrZccAcj5C5i0
 fractal_server/tasks/v2/local/reactivate.py,sha256=Q43DOadNeFyyfgNP67lUqaXmZsS6onv67XwxH_-5ANA,5756
 fractal_server/tasks/v2/local/reactivate_pixi.py,sha256=IuxDRaj8i6Rc582TIbc9HVKQ9pOvR4IepyXSaJx2PfQ,7565
 fractal_server/tasks/v2/ssh/__init__.py,sha256=vX5aIM9Hbn2T_cIP_LrZ5ekRqJzYm_GSfp-4Iv7kqeI,300
-fractal_server/tasks/v2/ssh/_utils.py,sha256=jMe3eg2duad8wtGS_RD6oOIvA0LyJEg5cn--qanGuwE,4954
+fractal_server/tasks/v2/ssh/_utils.py,sha256=7rEyykRbaGblunLhoDf0A3IF0aWPBX4wTd5ES3R42SM,5121
 fractal_server/tasks/v2/ssh/collect.py,sha256=WesUBtNaax9ST7CtgqAD2qd-ZII4t1JFPzRzltEOuM8,14333
-fractal_server/tasks/v2/ssh/collect_pixi.py,sha256=PWv-5cQFLA3n4NeFsTU-zfE1xWEy8LoLvHy41gRtWW8,14169
+fractal_server/tasks/v2/ssh/collect_pixi.py,sha256=unZXs3qjBoWYBIa2xWiIKz2CVMVBAhcloQ_9YJ97n5k,14211
 fractal_server/tasks/v2/ssh/deactivate.py,sha256=7gJ2mOah0wKwUnK0S9QpaXg08_WE95P0rC-oExAGhLE,12438
 fractal_server/tasks/v2/ssh/deactivate_pixi.py,sha256=K0yK_NPUqhFMj6cp6G_0Kfn0Yo7oQux4kT5dFPulnos,4748
 fractal_server/tasks/v2/ssh/reactivate.py,sha256=NJIgMNFKaXMhbvK0iZOsMwMtsms6Boj9f8N4L01X9Bo,8271
-fractal_server/tasks/v2/ssh/reactivate_pixi.py,sha256=JGP19nbmCCuE8Ya7sBfgrm78vORM0bF0tkt3bWakvF0,10281
+fractal_server/tasks/v2/ssh/reactivate_pixi.py,sha256=UOlG01wOv-eQCtUxSwphEY-Ey7OoUm5PMCSP9J-i1rY,10323
 fractal_server/tasks/v2/templates/1_create_venv.sh,sha256=PK0jdHKtQpda1zULebBaVPORt4t6V17wa4N1ohcj5ac,548
 fractal_server/tasks/v2/templates/2_pip_install.sh,sha256=jMJPQJXHKznO6fxOOXtFXKPdCmTf1VLLWj_JL_ZdKxo,1644
 fractal_server/tasks/v2/templates/3_pip_freeze.sh,sha256=JldREScEBI4cD_qjfX4UK7V4aI-FnX9ZvVNxgpSOBFc,168
@@ -230,8 +230,8 @@ fractal_server/types/validators/_workflow_task_arguments_validators.py,sha256=HL
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=Vn35lApt1T1J8nc09sAVqd10Cy0sa3dLipcljI-hkuk,2185
 fractal_server/zip_tools.py,sha256=H0w7wS5yE4ebj7hw1_77YQ959dl2c-L0WX6J_ro1TY4,4884
-fractal_server-2.15.4.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.15.4.dist-info/METADATA,sha256=xtZoBMySmrfFIEf-RXzQHmqZUUvZr2Ud1cyR1rMLqoM,4283
-fractal_server-2.15.4.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
-fractal_server-2.15.4.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.15.4.dist-info/RECORD,,
+fractal_server-2.15.6.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.15.6.dist-info/METADATA,sha256=2gU9d2hWDNEFSQLlfx119dh46y1qC8b8tmtWvX3k1pc,4283
+fractal_server-2.15.6.dist-info/WHEEL,sha256=7dDg4QLnNKTvwIDR9Ac8jJaAmBC_owJrckbC0jjThyA,88
+fractal_server-2.15.6.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.15.6.dist-info/RECORD,,