fractal-server 2.15.0a2__py3-none-any.whl → 2.15.0a4__py3-none-any.whl

fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -2,13 +2,13 @@ import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-from ....ssh._fabric import SingleUseFractalSSH
 from ..utils_background import fail_and_cleanup
 from ..utils_background import get_activity_and_task_group
 from ..utils_background import prepare_tasks_metadata
 from ..utils_database import create_db_tasks_and_update_task_group_sync
 from ..utils_pixi import parse_collect_stdout
 from ..utils_pixi import SOURCE_DIR_NAME
+from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.schemas.v2 import FractalUploadedFile
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
@@ -17,6 +17,7 @@ from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.config import get_settings
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import SingleUseFractalSSH
 from fractal_server.ssh._fabric import SSHConfig
 from fractal_server.syringe import Inject
 from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
@@ -64,41 +65,34 @@ def collect_ssh_pixi(
             logger_name=LOGGER_NAME,
             log_file_path=log_file_path,
         )
-        with SingleUseFractalSSH(
-            ssh_config=ssh_config,
-            logger_name=LOGGER_NAME,
-        ) as fractal_ssh:
-
-            with next(get_sync_db()) as db:
-                success, task_group, activity = get_activity_and_task_group(
-                    task_group_activity_id=task_group_activity_id,
-                    task_group_id=task_group_id,
-                    db=db,
-                )
-                if not success:
-                    return
+        logger.info("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
+
+            with SingleUseFractalSSH(
+                ssh_config=ssh_config,
+                logger_name=LOGGER_NAME,
+            ) as fractal_ssh:
 
-                # Log some info
-                logger.info("START")
-                for key, value in task_group.model_dump().items():
-                    logger.debug(f"task_group.{key}: {value}")
-
-                # Check that SSH connection works
                 try:
-                    fractal_ssh.check_connection()
-                except Exception as e:
-                    logger.error("Cannot establish SSH connection.")
-                    fail_and_cleanup(
+                    # Check SSH connection
+                    ssh_ok = check_ssh_or_fail_and_cleanup(
+                        fractal_ssh=fractal_ssh,
                         task_group=task_group,
                         task_group_activity=activity,
                         logger_name=LOGGER_NAME,
                         log_file_path=log_file_path,
-                        exception=e,
                         db=db,
                     )
-                    return
-
-                try:
+                    if not ssh_ok:
+                        return
 
                     # Check that the (remote) task_group path does not exist
                     if fractal_ssh.remote_exists(task_group.path):
@@ -115,11 +109,9 @@ def collect_ssh_pixi(
                         return
 
                     # Create remote `task_group.path` and `script_dir_remote`
-                    # folders (note that because of `parents=True` we  are in
-                    # the `no error if existing, make parent directories as
-                    # needed` scenario for `mkdir`)
-                    script_dir_remote = (
-                        Path(task_group.path) / SCRIPTS_SUBFOLDER
+                    # folders
+                    script_dir_remote = Path(
+                        task_group.path, SCRIPTS_SUBFOLDER
                     ).as_posix()
                     fractal_ssh.mkdir(folder=task_group.path, parents=True)
                     fractal_ssh.mkdir(folder=script_dir_remote, parents=True)
@@ -130,12 +122,8 @@ def collect_ssh_pixi(
                     archive_path = (
                         Path(task_group.path) / tar_gz_filename
                     ).as_posix()
-                    tmp_archive_path = (
-                        Path(tmpdir) / tar_gz_filename
-                    ).as_posix()
-                    logger.info(
-                        f"Write tar.gz-file contents into {tmp_archive_path}"
-                    )
+                    tmp_archive_path = Path(tmpdir, tar_gz_filename).as_posix()
+                    logger.info(f"Write tar.gz file into {tmp_archive_path}")
                     with open(tmp_archive_path, "wb") as f:
                         f.write(tar_gz_file.contents)
                     fractal_ssh.send_file(
@@ -145,19 +133,20 @@ def collect_ssh_pixi(
                     task_group.archive_path = archive_path
                     task_group = add_commit_refresh(obj=task_group, db=db)
 
-                    # Set `pixi_bin`
                     settings = Inject(get_settings)
-                    pixi_home = settings.pixi.versions[task_group.pixi_version]
-
                     replacements = {
-                        ("__PIXI_HOME__", pixi_home),
+                        (
+                            "__PIXI_HOME__",
+                            settings.pixi.versions[task_group.pixi_version],
+                        ),
                         ("__PACKAGE_DIR__", task_group.path),
-                        ("__TAR_GZ_PATH__", archive_path),
+                        ("__TAR_GZ_PATH__", task_group.archive_path),
                         (
                             "__IMPORT_PACKAGE_NAME__",
                             task_group.pkg_name.replace("-", "_"),
                         ),
                         ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
+                        ("__FROZEN_OPTION__", ""),
                     }
 
                     logger.info("installing - START")
@@ -169,15 +158,15 @@ def collect_ssh_pixi(
 
                     common_args = dict(
                         script_dir_local=(
-                            Path(tmpdir) / SCRIPTS_SUBFOLDER
+                            Path(tmpdir, SCRIPTS_SUBFOLDER)
                         ).as_posix(),
                         script_dir_remote=script_dir_remote,
                         prefix=(
                             f"{int(time.time())}_"
                             f"{TaskGroupActivityActionV2.COLLECT}"
                         ),
-                        fractal_ssh=fractal_ssh,
                         logger_name=LOGGER_NAME,
+                        fractal_ssh=fractal_ssh,
                     )
 
                     # Run the three pixi-related scripts
@@ -294,7 +283,7 @@ def collect_ssh_pixi(
                     except Exception as e_rm:
                         logger.error(
                             "Removing folder failed. "
-                            f"Original error:\n{str(e_rm)}"
+                            f"Original error: {str(e_rm)}"
                         )
                     fail_and_cleanup(
                         task_group=task_group,

fractal_server/tasks/v2/ssh/deactivate.py

@@ -8,6 +8,7 @@ from ..utils_background import get_activity_and_task_group
 from ..utils_templates import get_collection_replacements
 from ._utils import _copy_wheel_file_ssh
 from ._utils import _customize_and_run_template
+from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
@@ -53,41 +54,33 @@ def deactivate_ssh(
             logger_name=LOGGER_NAME,
             log_file_path=log_file_path,
         )
-        with SingleUseFractalSSH(
-            ssh_config=ssh_config,
-            logger_name=LOGGER_NAME,
-        ) as fractal_ssh:
-
-            with next(get_sync_db()) as db:
-                success, task_group, activity = get_activity_and_task_group(
-                    task_group_activity_id=task_group_activity_id,
-                    task_group_id=task_group_id,
-                    db=db,
-                )
-                if not success:
-                    return
-
-                # Log some info
-                logger.debug("START")
-                for key, value in task_group.model_dump().items():
-                    logger.debug(f"task_group.{key}: {value}")
-
-                # Check that SSH connection works
+        logger.debug("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
+
+            with SingleUseFractalSSH(
+                ssh_config=ssh_config,
+                logger_name=LOGGER_NAME,
+            ) as fractal_ssh:
                 try:
-                    fractal_ssh.check_connection()
-                except Exception as e:
-                    logger.error("Cannot establish SSH connection.")
-                    fail_and_cleanup(
+                    # Check SSH connection
+                    ssh_ok = check_ssh_or_fail_and_cleanup(
+                        fractal_ssh=fractal_ssh,
                         task_group=task_group,
                         task_group_activity=activity,
                         logger_name=LOGGER_NAME,
                         log_file_path=log_file_path,
-                        exception=e,
                         db=db,
                     )
-                    return
-
-                try:
+                    if not ssh_ok:
+                        return
 
                     # Check that the (local) task_group venv_path does exist
                     if not fractal_ssh.remote_exists(task_group.venv_path):

fractal_server/tasks/v2/ssh/deactivate_pixi.py

@@ -5,6 +5,7 @@ from ..utils_background import add_commit_refresh
 from ..utils_background import fail_and_cleanup
 from ..utils_background import get_activity_and_task_group
 from ..utils_pixi import SOURCE_DIR_NAME
+from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
 from fractal_server.logger import reset_logger_handlers
@@ -46,41 +47,34 @@ def deactivate_ssh_pixi(
             logger_name=LOGGER_NAME,
             log_file_path=log_file_path,
         )
-        with SingleUseFractalSSH(
-            ssh_config=ssh_config,
-            logger_name=LOGGER_NAME,
-        ) as fractal_ssh:
-
-            with next(get_sync_db()) as db:
-                success, task_group, activity = get_activity_and_task_group(
-                    task_group_activity_id=task_group_activity_id,
-                    task_group_id=task_group_id,
-                    db=db,
-                )
-                if not success:
-                    return
+        logger.debug("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
 
-                # Log some info
-                logger.debug("START")
-                for key, value in task_group.model_dump().items():
-                    logger.debug(f"task_group.{key}: {value}")
-
-                # Check that SSH connection works
+            with SingleUseFractalSSH(
+                ssh_config=ssh_config,
+                logger_name=LOGGER_NAME,
+            ) as fractal_ssh:
                 try:
-                    fractal_ssh.check_connection()
-                except Exception as e:
-                    logger.error("Cannot establish SSH connection.")
-                    fail_and_cleanup(
+                    # Check SSH connection
+                    ssh_ok = check_ssh_or_fail_and_cleanup(
+                        fractal_ssh=fractal_ssh,
                         task_group=task_group,
                         task_group_activity=activity,
                         logger_name=LOGGER_NAME,
                         log_file_path=log_file_path,
-                        exception=e,
                         db=db,
                     )
-                    return
+                    if not ssh_ok:
+                        return
 
-                try:
                     # Check that the (remote) task_group venv_path does exist
                     source_dir = Path(
                         task_group.path, SOURCE_DIR_NAME

fractal_server/tasks/v2/ssh/reactivate.py

@@ -7,6 +7,7 @@ from ..utils_background import fail_and_cleanup
 from ..utils_background import get_activity_and_task_group
 from ..utils_templates import get_collection_replacements
 from ._utils import _customize_and_run_template
+from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
@@ -54,41 +55,34 @@ def reactivate_ssh(
             log_file_path=log_file_path,
         )
 
-        with SingleUseFractalSSH(
-            ssh_config=ssh_config,
-            logger_name=LOGGER_NAME,
-        ) as fractal_ssh:
-
-            with next(get_sync_db()) as db:
-                success, task_group, activity = get_activity_and_task_group(
-                    task_group_activity_id=task_group_activity_id,
-                    task_group_id=task_group_id,
-                    db=db,
-                )
-                if not success:
-                    return
-
-                # Log some info
-                logger.info("START")
-                for key, value in task_group.model_dump().items():
-                    logger.debug(f"task_group.{key}: {value}")
-
-                # Check that SSH connection works
+        logger.info("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
+
+            with SingleUseFractalSSH(
+                ssh_config=ssh_config,
+                logger_name=LOGGER_NAME,
+            ) as fractal_ssh:
                 try:
-                    fractal_ssh.check_connection()
-                except Exception as e:
-                    logger.error("Cannot establish SSH connection.")
-                    fail_and_cleanup(
+                    # Check SSH connection
+                    ssh_ok = check_ssh_or_fail_and_cleanup(
+                        fractal_ssh=fractal_ssh,
                         task_group=task_group,
                         task_group_activity=activity,
                         logger_name=LOGGER_NAME,
                         log_file_path=log_file_path,
-                        exception=e,
                         db=db,
                     )
-                    return
+                    if not ssh_ok:
+                        return
 
-                try:
                     # Check that the (remote) task_group venv_path does not
                     # exist
                     if fractal_ssh.remote_exists(task_group.venv_path):
@@ -192,8 +186,8 @@ def reactivate_ssh(
                         logger.info(f"Deleted folder {task_group.venv_path}")
                     except Exception as rm_e:
                         logger.error(
-                            "Removing folder failed.\n"
-                            f"Original error:\n{str(rm_e)}"
+                            "Removing folder failed. "
+                            f"Original error: {str(rm_e)}"
                         )
 
                     fail_and_cleanup(

fractal_server/tasks/v2/ssh/reactivate_pixi.py

@@ -1,14 +1,26 @@
+import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..utils_background import fail_and_cleanup
 from ..utils_background import get_activity_and_task_group
+from ..utils_pixi import SOURCE_DIR_NAME
+from ._utils import check_ssh_or_fail_and_cleanup
 from fractal_server.app.db import get_sync_db
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.config import get_settings
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import SingleUseFractalSSH
 from fractal_server.ssh._fabric import SSHConfig
+from fractal_server.syringe import Inject
 from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
+from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import get_timestamp
 
 
 def reactivate_ssh_pixi(
@@ -42,60 +54,156 @@ def reactivate_ssh_pixi(
             log_file_path=log_file_path,
         )
 
-        with SingleUseFractalSSH(
-            ssh_config=ssh_config,
-            logger_name=LOGGER_NAME,
-        ) as fractal_ssh:
-
-            with next(get_sync_db()) as db:
-                success, task_group, activity = get_activity_and_task_group(
-                    task_group_activity_id=task_group_activity_id,
-                    task_group_id=task_group_id,
-                    db=db,
-                )
-                if not success:
-                    return
-
-                # Log some info
-                logger.info("START")
-                for key, value in task_group.model_dump().items():
-                    logger.debug(f"task_group.{key}: {value}")
-
-                # Check that SSH connection works
+        logger.info("START")
+        with next(get_sync_db()) as db:
+            db_objects_ok, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+                logger_name=LOGGER_NAME,
+            )
+            if not db_objects_ok:
+                return
+
+            with SingleUseFractalSSH(
+                ssh_config=ssh_config,
+                logger_name=LOGGER_NAME,
+            ) as fractal_ssh:
                 try:
-                    fractal_ssh.check_connection()
-                except Exception as e:
-                    logger.error("Cannot establish SSH connection.")
-                    fail_and_cleanup(
+                    # Check SSH connection
+                    ssh_ok = check_ssh_or_fail_and_cleanup(
+                        fractal_ssh=fractal_ssh,
                         task_group=task_group,
                         task_group_activity=activity,
                         logger_name=LOGGER_NAME,
                         log_file_path=log_file_path,
-                        exception=e,
                         db=db,
                     )
-                    return
+                    if not ssh_ok:
+                        return
 
-                try:
-                    raise NotImplementedError("pixi-task reactivation FIXME")
+                    # Check that the (remote) task_group source_dir does not
+                    # exist
+                    source_dir = Path(
+                        task_group.path, SOURCE_DIR_NAME
+                    ).as_posix()
+                    if fractal_ssh.remote_exists(source_dir):
+                        error_msg = f"{source_dir} already exists."
+                        logger.error(error_msg)
+                        fail_and_cleanup(
+                            task_group=task_group,
+                            task_group_activity=activity,
+                            logger_name=LOGGER_NAME,
+                            log_file_path=log_file_path,
+                            exception=FileExistsError(error_msg),
+                            db=db,
+                        )
+                        return
+
+                    settings = Inject(get_settings)
+                    replacements = {
+                        (
+                            "__PIXI_HOME__",
+                            settings.pixi.versions[task_group.pixi_version],
+                        ),
+                        ("__PACKAGE_DIR__", task_group.path),
+                        ("__TAR_GZ_PATH__", task_group.archive_path),
+                        (
+                            "__IMPORT_PACKAGE_NAME__",
+                            task_group.pkg_name.replace("-", "_"),
+                        ),
+                        ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
+                        ("__FROZEN_OPTION__", "--frozen"),
+                    }
+
+                    logger.info("installing - START")
+
+                    # Set status to ONGOING and refresh logs
+                    activity.status = TaskGroupActivityStatusV2.ONGOING
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    script_dir_remote = Path(
+                        task_group.path, SCRIPTS_SUBFOLDER
+                    ).as_posix()
+                    common_args = dict(
+                        script_dir_local=(
+                            Path(tmpdir) / SCRIPTS_SUBFOLDER
+                        ).as_posix(),
+                        script_dir_remote=script_dir_remote,
+                        prefix=(
+                            f"{int(time.time())}_"
+                            f"{TaskGroupActivityActionV2.REACTIVATE}"
+                        ),
+                        logger_name=LOGGER_NAME,
+                        fractal_ssh=fractal_ssh,
+                    )
+
+                    # Run script 1 - extract tar.gz into `source_dir`
+                    _customize_and_run_template(
+                        template_filename="pixi_1_extract.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    # Write pixi.lock into `source_dir`
+                    pixi_lock_local = Path(tmpdir, "pixi.lock").as_posix()
+                    pixi_lock_remote = Path(
+                        task_group.path, SOURCE_DIR_NAME, "pixi.lock"
+                    ).as_posix()
+                    logger.info(
+                        f"Write `env_info` contents into {pixi_lock_local}"
+                    )
+                    with open(pixi_lock_local, "w") as f:
+                        f.write(task_group.env_info)
+                    fractal_ssh.send_file(
+                        local=pixi_lock_local,
+                        remote=pixi_lock_remote,
+                    )
+
+                    # Run script 2 - run pixi-install command
+                    _customize_and_run_template(
+                        template_filename="pixi_2_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    # Run script 3 - post-install
+                    _customize_and_run_template(
+                        template_filename="pixi_3_post_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    # Finalize (write metadata to DB)
+                    activity.status = TaskGroupActivityStatusV2.OK
+                    activity.timestamp_ended = get_timestamp()
+                    activity = add_commit_refresh(obj=activity, db=db)
+                    task_group.active = True
+                    task_group = add_commit_refresh(obj=task_group, db=db)
+                    logger.info("END")
 
                     reset_logger_handlers(logger)
 
                 except Exception as reactivate_e:
-                    # Delete corrupted venv_path
+                    # Delete corrupted source_dir
                     try:
-                        logger.info(
-                            f"Now delete folder {task_group.venv_path}"
-                        )
+                        logger.info(f"Now delete folder {source_dir}")
                         fractal_ssh.remove_folder(
-                            folder=task_group.venv_path,
+                            folder=source_dir,
                             safe_root=tasks_base_dir,
                         )
-                        logger.info(f"Deleted folder {task_group.venv_path}")
+                        logger.info(f"Deleted folder {source_dir}")
                     except Exception as rm_e:
                         logger.error(
-                            "Removing folder failed.\n"
-                            f"Original error:\n{str(rm_e)}"
+                            "Removing folder failed. "
+                            f"Original error: {str(rm_e)}"
                         )
 
                     fail_and_cleanup(

fractal_server/tasks/v2/templates/pixi_2_install.sh

@@ -32,14 +32,9 @@ write_log "Changed working directory to ${PACKAGE_DIR}"
 
 # -----------------------------------------------------------------------------
 
-FROZEN_FLAG=""
-if [[ "${FROZEN_OPTION}" == "true" ]]; then
-  FROZEN_FLAG="--frozen"
-fi
-
-write_log "START '${PIXI_EXECUTABLE} install ${FROZEN_FLAG} --manifest-path ${PYPROJECT_TOML}'"
-${PIXI_EXECUTABLE} install ${FROZEN_FLAG} --manifest-path "${PYPROJECT_TOML}"
-write_log "END   '${PIXI_EXECUTABLE} install ${FROZEN_FLAG} --manifest-path ${PYPROJECT_TOML}'"
+write_log "START '${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path ${PYPROJECT_TOML}'"
+${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path "${PYPROJECT_TOML}"
+write_log "END   '${PIXI_EXECUTABLE} install ${FROZEN_OPTION} --manifest-path ${PYPROJECT_TOML}'"
 echo
 
 TIME_END=$(date +%s)

fractal_server/tasks/v2/utils_background.py

@@ -30,6 +30,7 @@ def get_activity_and_task_group(
     task_group_activity_id: int,
     task_group_id: int,
     db: DBSyncSession,
+    logger_name: str,
 ) -> tuple[bool, TaskGroupV2, TaskGroupActivityV2]:
     task_group = db.get(TaskGroupV2, task_group_id)
     activity = db.get(TaskGroupActivityV2, task_group_activity_id)
@@ -40,6 +41,12 @@ def get_activity_and_task_group(
             f"{task_group=}\n{activity=}. Exit."
         )
         return False, None, None
+
+    # Log some info about task group
+    logger = get_logger(logger_name=logger_name)
+    for key, value in task_group.model_dump(exclude={"env_info"}).items():
+        logger.debug(f"task_group.{key}: {value}")
+
     return True, task_group, activity