fractal-server 2.16.2a0__tar.gz → 2.16.3__tar.gz

{fractal_server-2.16.2a0 → fractal_server-2.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.16.2a0
+Version: 2.16.3
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: alembic (>=1.13.1,<2.0.0)
-Requires-Dist: cryptography (>=45.0.3,<45.1.0)
+Requires-Dist: cryptography (>=46.0.0,<47.0.0)
 Requires-Dist: fabric (>=3.2.2,<3.3.0)
 Requires-Dist: fastapi (>=0.116.0,<0.117.0)
 Requires-Dist: fastapi-users[oauth] (>=14,<15)

fractal_server-2.16.3/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.16.3"

fractal_server-2.16.3/fractal_server/tasks/v2/ssh/_pixi_slurm_ssh.py

@@ -0,0 +1,255 @@
+import os
+import time
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from fractal_server.app.models.v2 import TaskGroupActivityV2
+from fractal_server.config import get_settings
+from fractal_server.config import PixiSLURMConfig
+from fractal_server.logger import get_logger
+from fractal_server.ssh._fabric import FractalSSH
+from fractal_server.syringe import Inject
+from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import get_current_log
+
+FRACTAL_SQUEUE_ERROR_STATE = "__FRACTAL_SQUEUE_ERROR__"
+
+# https://slurm.schedmd.com/squeue.html#lbAG
+STATES_FINISHED = {
+    "BOOT_FAIL",
+    "CANCELLED",
+    "COMPLETED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "SPECIAL_EXIT",
+    "TIMEOUT",
+    FRACTAL_SQUEUE_ERROR_STATE,
+}
+
+
+def _get_workdir_remote(script_paths: list[str]) -> str:
+    """
+    Check that there is one and only one `workdir`, and return it.
+
+    Note: The `is_absolute` check is to filter out a `chmod` command.
+    """
+    workdirs = [
+        Path(script_path).parent.as_posix()
+        for script_path in script_paths
+        if Path(script_path).is_absolute()
+    ]
+    if not len(set(workdirs)) == 1:
+        raise ValueError(f"Invalid {script_paths=}.")
+    return workdirs[0]
+
+
+def _read_file_if_exists(
+    *,
+    fractal_ssh: FractalSSH,
+    path: str,
+) -> str:
+    """
+    Read a remote file if it exists, or return an empty string.
+    """
+    if fractal_ssh.remote_exists(path=path):
+        return fractal_ssh.read_remote_text_file(path)
+    else:
+        return ""
+
+
+def _log_change_of_job_state(
+    *,
+    old_state: str | None,
+    new_state: str,
+    logger_name: str,
+) -> None:
+    """
+    Emit a log for state changes.
+
+    Args:
+        old_state:
+        new_state:
+        logger_name:
+    """
+    if new_state != old_state:
+        logger = get_logger(logger_name=logger_name)
+        logger.debug(
+            f"SLURM-job state changed from {old_state=} to {new_state=}."
+        )
+
+
+def _run_squeue(
+    *,
+    fractal_ssh: FractalSSH,
+    squeue_cmd: str,
+    logger_name: str,
+) -> str:
+    """
+    Run a `squeue` command and handle exceptions.
+
+    Args:
+        fractal_ssh:
+        logger_name:
+        squeue_cmd:
+
+    Return:
+        state: The SLURM-job state.
+    """
+    try:
+        cmd_stdout = fractal_ssh.run_command(cmd=squeue_cmd)
+        state = cmd_stdout.strip().split()[1]
+        return state
+    except Exception as e:
+        logger = get_logger(logger_name=logger_name)
+        logger.info(f"`squeue` command failed (original error: {e})")
+        return FRACTAL_SQUEUE_ERROR_STATE
+
+
+def _verify_success_file_exists(
+    *,
+    fractal_ssh: FractalSSH,
+    success_file_remote: str,
+    logger_name: str,
+    stderr_remote: str,
+) -> None:
+    """
+    Fail if the success sentinel file does not exist remotely.
+
+    Note: the `FractalSSH` methods in this function may fail, and such failures
+    are not handled in this function. Any such failure, however, will lead to
+    a "failed" task-group lifecycle activity (because it will raise an
+    exception from within `run_script_on_remote_slurm`, which will then be
+    handled at the calling-function level.
+    """
+    if not fractal_ssh.remote_exists(path=success_file_remote):
+        logger = get_logger(logger_name=logger_name)
+        error_msg = f"{success_file_remote=} missing."
+        logger.info(error_msg)
+
+        stderr = _read_file_if_exists(
+            fractal_ssh=fractal_ssh, path=stderr_remote
+        )
+        if stderr:
+            logger.info(f"SLURM-job stderr:\n{stderr}")
+        raise RuntimeError(error_msg)
+
+
+def run_script_on_remote_slurm(
+    *,
+    script_paths: list[str],
+    slurm_config: PixiSLURMConfig,
+    fractal_ssh: FractalSSH,
+    logger_name: str,
+    log_file_path: Path,
+    prefix: str,
+    db: Session,
+    activity: TaskGroupActivityV2,
+):
+    """
+    Run a `pixi install` script as a SLURM job.
+
+    NOTE: This is called from within a try/except, thus we can use exceptions
+    as a mechanism to propagate failure/errors.
+    """
+
+    logger = get_logger(logger_name=logger_name)
+    settings = Inject(get_settings)
+
+    # (1) Prepare remote submission script
+    workdir_remote = _get_workdir_remote(script_paths)
+    submission_script_remote = os.path.join(
+        workdir_remote, f"{prefix}-submit.sh"
+    )
+    stderr_remote = os.path.join(workdir_remote, f"{prefix}-err.txt")
+    stdout_remote = os.path.join(workdir_remote, f"{prefix}-out.txt")
+    success_file_remote = os.path.join(workdir_remote, f"{prefix}-success.txt")
+    script_lines = [
+        "#!/bin/bash",
+        f"#SBATCH --partition={slurm_config.partition}",
+        f"#SBATCH --cpus-per-task={slurm_config.cpus}",
+        f"#SBATCH --mem={slurm_config.mem}",
+        f"#SBATCH --time={slurm_config.time}",
+        f"#SBATCH --err={stderr_remote}",
+        f"#SBATCH --out={stdout_remote}",
+        f"#SBATCH -D {workdir_remote}",
+        "",
+    ]
+    for script_path in script_paths:
+        script_lines.append(f"bash {script_path}")
+    script_lines.append(f"touch {success_file_remote}")
+
+    script_contents = "\n".join(script_lines)
+    fractal_ssh.write_remote_file(
+        path=submission_script_remote,
+        content=script_contents,
+    )
+    logger.debug(f"Written {submission_script_remote=}.")
+
+    activity.log = get_current_log(log_file_path)
+    activity = add_commit_refresh(obj=activity, db=db)
+
+    # (2) Submit SLURM job
+    logger.debug("Now submit SLURM job.")
+    sbatch_cmd = f"sbatch --parsable {submission_script_remote}"
+    try:
+        stdout = fractal_ssh.run_command(cmd=sbatch_cmd)
+        job_id = int(stdout)
+        logger.debug(f"SLURM-job submission successful ({job_id=}).")
+    except Exception as e:
+        logger.error(
+            (
+                f"Submission of {submission_script_remote} failed. "
+                f"Original error: {str(e)}"
+            )
+        )
+        raise e
+    finally:
+        activity.log = get_current_log(log_file_path)
+        activity = add_commit_refresh(obj=activity, db=db)
+
+    # (3) Monitor job
+    squeue_cmd = (
+        f"squeue --noheader --format='%i %T' --states=all --jobs={job_id}"
+    )
+    logger.debug(f"Start monitoring job with {squeue_cmd=}.")
+    old_state = None
+    while True:
+        new_state = _run_squeue(
+            fractal_ssh=fractal_ssh,
+            squeue_cmd=squeue_cmd,
+            logger_name=logger_name,
+        )
+        _log_change_of_job_state(
+            old_state=old_state,
+            new_state=new_state,
+            logger_name=logger_name,
+        )
+        activity.log = get_current_log(log_file_path)
+        activity = add_commit_refresh(obj=activity, db=db)
+        if new_state in STATES_FINISHED:
+            logger.debug(f"Exit retrieval loop (state={new_state}).")
+            break
+        old_state = new_state
+        time.sleep(settings.FRACTAL_SLURM_POLL_INTERVAL)
+
+    _verify_success_file_exists(
+        fractal_ssh=fractal_ssh,
+        logger_name=logger_name,
+        success_file_remote=success_file_remote,
+        stderr_remote=stderr_remote,
+    )
+
+    stdout = _read_file_if_exists(
+        fractal_ssh=fractal_ssh,
+        path=stdout_remote,
+    )
+
+    logger.info("SLURM-job execution completed successfully, continue.")
+    activity.log = get_current_log(log_file_path)
+    activity = add_commit_refresh(obj=activity, db=db)
+
+    return stdout

{fractal_server-2.16.2a0 → fractal_server-2.16.3}/fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -203,12 +203,21 @@ def collect_ssh_pixi(
                         pyproject_toml_path=pyproject_toml_path,
                     )
 
-                    # Run script 2 - run pixi-install command
+                    # Prepare scripts 2 and 3
                     remote_script2_path = _customize_and_send_template(
                         template_filename="pixi_2_install.sh",
                         replacements=replacements,
                         **common_args,
                     )
+                    remote_script3_path = _customize_and_send_template(
+                        template_filename="pixi_3_post_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    logger.debug(
+                        "Post-installation script written to "
+                        f"{remote_script3_path=}."
+                    )
                     logger.debug(
                         "Installation script written to "
                         f"{remote_script2_path=}."
@@ -216,26 +225,24 @@ def collect_ssh_pixi(
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
-                    run_script_on_remote_slurm(
-                        script_path=remote_script2_path,
+                    # Run scripts 2 and 3
+                    stdout = run_script_on_remote_slurm(
+                        script_paths=[
+                            remote_script2_path,
+                            remote_script3_path,
+                            f"chmod -R 755 {source_dir}",
+                        ],
                         slurm_config=settings.pixi.SLURM_CONFIG,
                         fractal_ssh=fractal_ssh,
                         logger_name=LOGGER_NAME,
                         prefix=common_args["prefix"],
+                        db=db,
+                        activity=activity,
+                        log_file_path=log_file_path,
                     )
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
-                    # Run script 3 - post-install
-                    stdout = _customize_and_run_template(
-                        template_filename="pixi_3_post_install.sh",
-                        replacements=replacements,
-                        **common_args,
-                    )
-                    logger.debug(f"STDOUT: {stdout}")
-                    activity.log = get_current_log(log_file_path)
-                    activity = add_commit_refresh(obj=activity, db=db)
-
                     # Parse stdout
                     parsed_output = parse_collect_stdout(stdout)
                     package_root_remote = parsed_output["package_root"]
@@ -245,8 +252,6 @@ def collect_ssh_pixi(
                         "project_python_wrapper"
                     ]
 
-                    fractal_ssh.run_command(cmd=f"chmod -R 755 {source_dir}")
-
                     # Read and validate remote manifest file
                     manifest_path_remote = (
                         f"{package_root_remote}/__FRACTAL_MANIFEST__.json"

{fractal_server-2.16.2a0 → fractal_server-2.16.3}/fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -188,12 +188,21 @@ def reactivate_ssh_pixi(
                         remote=pixi_lock_remote,
                     )
 
-                    # Run script 2 - run pixi-install command
+                    # Prepare scripts 2 and 3
                     remote_script2_path = _customize_and_send_template(
                         template_filename="pixi_2_install.sh",
                         replacements=replacements,
                         **common_args,
                     )
+                    remote_script3_path = _customize_and_send_template(
+                        template_filename="pixi_3_post_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    logger.debug(
+                        "Post-installation script written to "
+                        f"{remote_script3_path=}."
+                    )
                     logger.debug(
                         "Installation script written to "
                         f"{remote_script2_path=}."
@@ -201,28 +210,24 @@ def reactivate_ssh_pixi(
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
-                    run_script_on_remote_slurm(
-                        script_path=remote_script2_path,
+                    # Run scripts 2 and 3
+                    stdout = run_script_on_remote_slurm(
+                        script_paths=[
+                            remote_script2_path,
+                            remote_script3_path,
+                            f"chmod -R 755 {source_dir}",
+                        ],
                         slurm_config=settings.pixi.SLURM_CONFIG,
                         fractal_ssh=fractal_ssh,
                         logger_name=LOGGER_NAME,
                         prefix=common_args["prefix"],
+                        db=db,
+                        activity=activity,
+                        log_file_path=log_file_path,
                     )
                     activity.log = get_current_log(log_file_path)
                     activity = add_commit_refresh(obj=activity, db=db)
 
-                    # Run script 3 - post-install
-                    stdout = _customize_and_run_template(
-                        template_filename="pixi_3_post_install.sh",
-                        replacements=replacements,
-                        **common_args,
-                    )
-                    logger.debug(f"STDOUT: {stdout}")
-                    activity.log = get_current_log(log_file_path)
-                    activity = add_commit_refresh(obj=activity, db=db)
-
-                    fractal_ssh.run_command(cmd=f"chmod -R 755 {source_dir}")
-
                     # Finalize (write metadata to DB)
                     activity.status = TaskGroupActivityStatusV2.OK
                     activity.timestamp_ended = get_timestamp()

{fractal_server-2.16.2a0 → fractal_server-2.16.3}/fractal_server/tasks/v2/utils_background.py

@@ -143,6 +143,6 @@ def prepare_tasks_metadata(
     return task_list
 
 
-def get_current_log(logger_file_path: str) -> str:
+def get_current_log(logger_file_path: str | Path) -> str:
     with open(logger_file_path) as f:
         return f.read()

{fractal_server-2.16.2a0 → fractal_server-2.16.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.16.2a0"
+version = "2.16.3"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -26,7 +26,7 @@ dependencies = [
     "fabric >= 3.2.2, <3.3.0",
     "gunicorn >=23.0,<24.0",
     "psycopg[binary] >= 3.1.0, <4.0.0",
-    "cryptography >=45.0.3,<45.1.0",
+    "cryptography >=46.0.0,<47.0.0",
     "tomli_w >=1.2.0, <1.3.0 "
 ]
 
@@ -95,7 +95,7 @@ filterwarnings = [
 markers = ["container", "ssh", "fails_on_macos"]
 
 [tool.bumpver]
-current_version = "2.16.2a0"
+current_version = "2.16.3"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.16.2a0/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.16.2a0"

fractal_server-2.16.2a0/fractal_server/tasks/v2/ssh/_pixi_slurm_ssh.py

@@ -1,115 +0,0 @@
-import os
-import time
-from pathlib import Path
-
-from fractal_server.config import get_settings
-from fractal_server.config import PixiSLURMConfig
-from fractal_server.logger import get_logger
-from fractal_server.ssh._fabric import FractalSSH
-from fractal_server.syringe import Inject
-
-
-# https://slurm.schedmd.com/squeue.html#lbAG
-STATES_FINISHED = {
-    "BOOT_FAIL",
-    "CANCELLED",
-    "COMPLETED",
-    "DEADLINE",
-    "FAILED",
-    "NODE_FAIL",
-    "OUT_OF_MEMORY",
-    "PREEMPTED",
-    "SPECIAL_EXIT",
-    "TIMEOUT",
-}
-
-
-def run_script_on_remote_slurm(
-    *,
-    script_path: str,
-    slurm_config: PixiSLURMConfig,
-    fractal_ssh: FractalSSH,
-    logger_name: str,
-    prefix: str,
-):
-    """
-    FIXME
-
-
-    NOTE: This is called from within a try/except, thus we can use exceptions
-    as a mechanism to propagate failure/errors.
-    """
-
-    logger = get_logger(logger_name=logger_name)
-    settings = Inject(get_settings)
-
-    # (1) Prepare remote submission script
-    workdir_remote = Path(script_path).parent.as_posix()
-    submission_script_remote = os.path.join(
-        workdir_remote, f"{prefix}-submit.sh"
-    )
-    stderr_remote = os.path.join(workdir_remote, f"{prefix}-err.txt")
-    stdout_remote = os.path.join(workdir_remote, f"{prefix}-out.txt")
-    success_file_remote = os.path.join(workdir_remote, f"{prefix}-success.txt")
-    script_lines = [
-        "#!/bin/bash",
-        f"#SBATCH --partition={slurm_config.partition}",
-        f"#SBATCH --cpus-per-task={slurm_config.cpus}",
-        f"#SBATCH --mem={slurm_config.mem}",
-        f"#SBATCH --time={slurm_config.time}",
-        f"#SBATCH --err={stderr_remote}",
-        f"#SBATCH --out={stdout_remote}",
-        f"#SBATCH -D {workdir_remote}",
-        "",
-        f"bash {script_path}",
-        f"touch {success_file_remote}",
-        "",
-    ]
-    script_contents = "\n".join(script_lines)
-    fractal_ssh.write_remote_file(
-        path=submission_script_remote,
-        content=script_contents,
-    )
-
-    # (2) Submit SLURM job
-    sbatch_cmd = f"sbatch --parsable {submission_script_remote} "
-    try:
-        stdout = fractal_ssh.run_command(cmd=sbatch_cmd)
-    except Exception as e:
-        logger.error(
-            f"Submission of {submission_script_remote} failed. "
-            f"Original error: {str(e)}"
-        )
-        raise e
-    logger.debug(f"Now submit job {submission_script_remote} to SLURM.")
-    job_id = int(stdout)
-    logger.debug(f"SLURM-job submission successful ({job_id=}).")
-
-    # (3) Monitor job
-    squeue_cmd = (
-        f"squeue --noheader --format='%i %T' --states=all --jobs={job_id}"
-    )
-    while True:
-        try:
-            stdout = fractal_ssh.run_command(cmd=squeue_cmd)
-        except Exception as e:
-            # FIXME: review this logic
-            logger.info(
-                f"`squeue` command failed (original error: {e}), "
-                "consider the job as complete."
-            )
-            break
-        state = stdout.strip().split()[1]
-        logger.debug(f"Status of SLURM job {job_id}: {state}")
-        if state in STATES_FINISHED:
-            logger.debug(f"Exit retrieval loop ({state=}).")
-            break
-        time.sleep(settings.FRACTAL_SLURM_POLL_INTERVAL)
-
-    if fractal_ssh.remote_exists(path=success_file_remote):
-        logger.info(f"{success_file_remote=} exists.")
-    else:
-        raise RuntimeError(
-            "SLURM job did not complete correctly "
-            f"({success_file_remote=} missing)."
-        )