fractal-server 2.14.16__py3-none-any.whl → 2.15.0a1__py3-none-any.whl

fractal_server/tasks/v2/local/reactivate.py

@@ -1,4 +1,3 @@
-import logging
 import shutil
 import time
 from pathlib import Path
@@ -6,11 +5,10 @@ from tempfile import TemporaryDirectory
 
 from ..utils_background import add_commit_refresh
 from ..utils_background import fail_and_cleanup
+from ..utils_background import get_activity_and_task_group
 from ..utils_templates import get_collection_replacements
 from ._utils import _customize_and_run_template
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.models.v2 import TaskGroupActivityV2
-from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
 from fractal_server.logger import reset_logger_handlers
@@ -50,17 +48,12 @@ def reactivate_local(
         )
 
         with next(get_sync_db()) as db:
-
-            # Get main objects from db
-            activity = db.get(TaskGroupActivityV2, task_group_activity_id)
-            task_group = db.get(TaskGroupV2, task_group_id)
-            if activity is None or task_group is None:
-                # Use `logging` directly
-                logging.error(
-                    "Cannot find database rows with "
-                    f"{task_group_id=} and {task_group_activity_id=}:\n"
-                    f"{task_group=}\n{activity=}. Exit."
-                )
+            success, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+            )
+            if not success:
                 return
 
             # Log some info
@@ -95,11 +88,11 @@ def reactivate_local(
                     ),
                 )
                 with open(f"{tmpdir}/pip_freeze.txt", "w") as f:
-                    f.write(task_group.pip_freeze)
+                    f.write(task_group.env_info)
                 replacements.append(
                     ("__PIP_FREEZE_FILE__", f"{tmpdir}/pip_freeze.txt")
                 )
-                # Prepare common arguments for `_customize_and_run_template``
+                # Prepare common arguments for `_customize_and_run_template`
                 common_args = dict(
                     replacements=replacements,
                     script_dir=(

fractal_server/tasks/v2/local/reactivate_pixi.py

@@ -0,0 +1,146 @@
+import shlex
+import shutil
+import subprocess  # nosec
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ..utils_background import add_commit_refresh
+from ..utils_background import fail_and_cleanup
+from ..utils_background import get_activity_and_task_group
+from ..utils_pixi import SOURCE_DIR_NAME
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.schemas.v2.task_group import TaskGroupActivityStatusV2
+from fractal_server.config import get_settings
+from fractal_server.logger import reset_logger_handlers
+from fractal_server.logger import set_logger
+from fractal_server.syringe import Inject
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.utils import get_timestamp
+
+
+def reactivate_local_pixi(
+    *,
+    task_group_activity_id: int,
+    task_group_id: int,
+) -> None:
+    """
+    Reactivate a task group venv.
+
+    This function is run as a background task, therefore exceptions must be
+    handled.
+
+    Arguments:
+        task_group_id:
+        task_group_activity_id:
+    """
+
+    LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
+
+    with TemporaryDirectory() as tmpdir:
+        log_file_path = get_log_path(Path(tmpdir))
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+        with next(get_sync_db()) as db:
+            success, task_group, activity = get_activity_and_task_group(
+                task_group_activity_id=task_group_activity_id,
+                task_group_id=task_group_id,
+                db=db,
+            )
+            if not success:
+                return
+
+            # Log some info
+            logger.debug("START")
+
+            for key, value in task_group.model_dump().items():
+                logger.debug(f"task_group.{key}: {value}")
+
+            source_dir = Path(task_group.path, SOURCE_DIR_NAME).as_posix()
+            if Path(source_dir).exists():
+                error_msg = f"{source_dir} already exists."
+                logger.error(error_msg)
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=FileExistsError(error_msg),
+                    db=db,
+                )
+                return
+
+            try:
+                activity.status = TaskGroupActivityStatusV2.ONGOING
+                activity = add_commit_refresh(obj=activity, db=db)
+
+                logger.debug("start - writing pixi lock")
+                with open(f"{task_group.path}/pixi.lock", "w") as f:
+                    f.write(task_group.env_info)
+                logger.debug("end - writing pixi lock")
+
+                subprocess.run(  # nosec
+                    shlex.split(
+                        f"tar xz -f {task_group.archive_path} "
+                        f"{Path(task_group.archive_path).name}"
+                    ),
+                    encoding="utf-8",
+                    cwd=task_group.path,
+                )
+
+                subprocess.run(  # nosec
+                    shlex.split(
+                        f"mv {Path(task_group.archive_path).name} {source_dir}"
+                    ),
+                    encoding="utf-8",
+                    cwd=task_group.path,
+                )
+
+                settings = Inject(get_settings)
+                pixi_home = settings.pixi.versions[task_group.pixi_version]
+                pixi_bin = Path(pixi_home, "bin/pixi").as_posix()
+
+                logger.debug("start - pixi install")
+                subprocess.run(  # nosec
+                    shlex.split(
+                        f"{pixi_bin} install "
+                        f"--manifest-path {source_dir}/pyproject.toml --frozen"
+                    ),
+                    encoding="utf-8",
+                    cwd=task_group.path,
+                )
+                logger.debug("end - pixi install")
+
+                activity.log = get_current_log(log_file_path)
+                activity.status = TaskGroupActivityStatusV2.OK
+                activity.timestamp_ended = get_timestamp()
+                activity = add_commit_refresh(obj=activity, db=db)
+                task_group.active = True
+                task_group = add_commit_refresh(obj=task_group, db=db)
+                logger.debug("END")
+
+                reset_logger_handlers(logger)
+
+            except Exception as reactivate_e:
+                # Delete corrupted source_dir
+                try:
+                    logger.info(f"Now delete folder {source_dir}")
+                    shutil.rmtree(source_dir)
+                    logger.info(f"Deleted folder {source_dir}")
+                except Exception as rm_e:
+                    logger.error(
+                        "Removing folder failed.\n"
+                        f"Original error:\n{str(rm_e)}"
+                    )
+
+                fail_and_cleanup(
+                    task_group=task_group,
+                    task_group_activity=activity,
+                    logger_name=LOGGER_NAME,
+                    log_file_path=log_file_path,
+                    exception=reactivate_e,
+                    db=db,
+                )
+        return

fractal_server/tasks/v2/ssh/__init__.py

@@ -1,3 +1,6 @@
 from .collect import collect_ssh  # noqa
+from .collect_pixi import collect_ssh_pixi  # noqa
 from .deactivate import deactivate_ssh  # noqa
+from .deactivate_pixi import deactivate_ssh_pixi  # noqa
 from .reactivate import reactivate_ssh  # noqa
+from .reactivate_pixi import reactivate_ssh_pixi  # noqa

fractal_server/tasks/v2/ssh/_utils.py

@@ -69,16 +69,16 @@ def _copy_wheel_file_ssh(
     *, task_group: TaskGroupV2, fractal_ssh: FractalSSH, logger_name: str
 ) -> str:
     """
-    Handle the situation where `task_group.wheel_path` is not part of
-    `task_group.path`, by copying `wheel_path` into `path`.
+    Handle the situation where `task_group.archive_path` is not part of
+    `task_group.path`, by copying `archive_path` into `path`.
 
     Returns:
-        The new `wheel_path`.
+        The new `archive_path`.
     """
     logger = get_logger(logger_name=logger_name)
-    source = task_group.wheel_path
+    source = task_group.archive_path
     dest = (
-        Path(task_group.path) / Path(task_group.wheel_path).name
+        Path(task_group.path) / Path(task_group.archive_path).name
     ).as_posix()
     cmd = f"cp {source} {dest}"
     logger.debug(f"[_copy_wheel_file] START {source=} {dest=}")

fractal_server/tasks/v2/ssh/collect.py

@@ -1,18 +1,16 @@
-import logging
 import time
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ....ssh._fabric import SingleUseFractalSSH
-from ..utils_background import _prepare_tasks_metadata
 from ..utils_background import fail_and_cleanup
+from ..utils_background import get_activity_and_task_group
+from ..utils_background import prepare_tasks_metadata
 from ..utils_database import create_db_tasks_and_update_task_group_sync
 from fractal_server.app.db import get_sync_db
-from fractal_server.app.models.v2 import TaskGroupActivityV2
-from fractal_server.app.models.v2 import TaskGroupV2
+from fractal_server.app.schemas.v2 import FractalUploadedFile
 from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
 from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
-from fractal_server.app.schemas.v2 import WheelFile
 from fractal_server.app.schemas.v2.manifest import ManifestV2
 from fractal_server.logger import reset_logger_handlers
 from fractal_server.logger import set_logger
@@ -38,7 +36,7 @@ def collect_ssh(
     task_group_activity_id: int,
     ssh_config: SSHConfig,
     tasks_base_dir: str,
-    wheel_file: WheelFile | None = None,
+    wheel_file: FractalUploadedFile | None = None,
 ) -> None:
     """
     Collect a task package over SSH
@@ -76,16 +74,12 @@ def collect_ssh(
         ) as fractal_ssh:
 
             with next(get_sync_db()) as db:
-                # Get main objects from db
-                activity = db.get(TaskGroupActivityV2, task_group_activity_id)
-                task_group = db.get(TaskGroupV2, task_group_id)
-                if activity is None or task_group is None:
-                    # Use `logging` directly
-                    logging.error(
-                        "Cannot find database rows with "
-                        f"{task_group_id=} and {task_group_activity_id=}:\n"
-                        f"{task_group=}\n{activity=}. Exit."
-                    )
+                success, task_group, activity = get_activity_and_task_group(
+                    task_group_activity_id=task_group_activity_id,
+                    task_group_id=task_group_id,
+                    db=db,
+                )
+                if not success:
                     return
 
                 # Log some info
@@ -135,25 +129,26 @@ def collect_ssh(
                     fractal_ssh.mkdir(folder=script_dir_remote, parents=True)
 
                     # Write wheel file locally and send it to remote path,
-                    # and set task_group.wheel_path
+                    # and set task_group.archive_path
                     if wheel_file is not None:
                         wheel_filename = wheel_file.filename
-                        wheel_path = (
+                        archive_path = (
                             Path(task_group.path) / wheel_filename
                         ).as_posix()
-                        tmp_wheel_path = (
+                        tmp_archive_path = (
                             Path(tmpdir) / wheel_filename
                         ).as_posix()
                         logger.info(
-                            f"Write wheel-file contents into {tmp_wheel_path}"
+                            "Write wheel-file contents into "
+                            f"{tmp_archive_path}"
                         )
-                        with open(tmp_wheel_path, "wb") as f:
+                        with open(tmp_archive_path, "wb") as f:
                             f.write(wheel_file.contents)
                         fractal_ssh.send_file(
-                            local=tmp_wheel_path,
-                            remote=wheel_path,
+                            local=tmp_archive_path,
+                            remote=archive_path,
                         )
-                        task_group.wheel_path = wheel_path
+                        task_group.archive_path = archive_path
                         task_group = add_commit_refresh(obj=task_group, db=db)
 
                     replacements = get_collection_replacements(
@@ -263,7 +258,7 @@ def collect_ssh(
                     logger.info("Manifest is a valid ManifestV2")
 
                     logger.info("_prepare_tasks_metadata - start")
-                    task_list = _prepare_tasks_metadata(
+                    task_list = prepare_tasks_metadata(
                         package_manifest=pkg_manifest,
                         package_version=task_group.version,
                         package_root=Path(package_root_remote),
@@ -283,15 +278,15 @@ def collect_ssh(
 
                     # Update task_group data
                     logger.info(
-                        "Add pip_freeze, venv_size and venv_file_number "
+                        "Add env_info, venv_size and venv_file_number "
                         "to TaskGroupV2 - start"
                     )
-                    task_group.pip_freeze = pip_freeze_stdout
+                    task_group.env_info = pip_freeze_stdout
                     task_group.venv_size_in_kB = int(venv_size)
                     task_group.venv_file_number = int(venv_file_number)
                     task_group = add_commit_refresh(obj=task_group, db=db)
                     logger.info(
-                        "Add pip_freeze, venv_size and venv_file_number "
+                        "Add env_info, venv_size and venv_file_number "
                         "to TaskGroupV2 - end"
                     )
 

fractal_server/tasks/v2/ssh/collect_pixi.py

@@ -0,0 +1,306 @@
+import time
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ....ssh._fabric import SingleUseFractalSSH
+from ..utils_background import fail_and_cleanup
+from ..utils_background import get_activity_and_task_group
+from ..utils_background import prepare_tasks_metadata
+from ..utils_database import create_db_tasks_and_update_task_group_sync
+from ..utils_pixi import parse_collect_stdout
+from ..utils_pixi import SOURCE_DIR_NAME
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.schemas.v2 import FractalUploadedFile
+from fractal_server.app.schemas.v2 import TaskGroupActivityActionV2
+from fractal_server.app.schemas.v2 import TaskGroupActivityStatusV2
+from fractal_server.app.schemas.v2.manifest import ManifestV2
+from fractal_server.config import get_settings
+from fractal_server.logger import reset_logger_handlers
+from fractal_server.logger import set_logger
+from fractal_server.ssh._fabric import SSHConfig
+from fractal_server.syringe import Inject
+from fractal_server.tasks.v2.ssh._utils import _customize_and_run_template
+from fractal_server.tasks.v2.utils_background import add_commit_refresh
+from fractal_server.tasks.v2.utils_background import get_current_log
+from fractal_server.tasks.v2.utils_templates import SCRIPTS_SUBFOLDER
+from fractal_server.utils import get_timestamp
+
+
+def collect_ssh_pixi(
+    *,
+    task_group_id: int,
+    task_group_activity_id: int,
+    ssh_config: SSHConfig,
+    tasks_base_dir: str,
+    tar_gz_file: FractalUploadedFile,
+) -> None:
+    """
+    Collect a task package over SSH
+
+    This function runs as a background task, therefore exceptions must be
+    handled.
+
+    NOTE: since this function is sync, it runs within a thread - due to
+    starlette/fastapi handling of background tasks (see
+    https://github.com/encode/starlette/blob/master/starlette/background.py).
+
+
+    Arguments:
+        task_group_id:
+        task_group_activity_id:
+        ssh_config:
+        tasks_base_dir:
+            Only used as a `safe_root` in `remove_dir`, and typically set to
+            `user_settings.ssh_tasks_dir`.
+        tar_gz_file:
+    """
+
+    LOGGER_NAME = f"{__name__}.ID{task_group_activity_id}"
+
+    # Work within a temporary folder, where also logs will be placed
+    with TemporaryDirectory() as tmpdir:
+        log_file_path = Path(tmpdir) / "log"
+        logger = set_logger(
+            logger_name=LOGGER_NAME,
+            log_file_path=log_file_path,
+        )
+        with SingleUseFractalSSH(
+            ssh_config=ssh_config,
+            logger_name=LOGGER_NAME,
+        ) as fractal_ssh:
+
+            with next(get_sync_db()) as db:
+                success, task_group, activity = get_activity_and_task_group(
+                    task_group_activity_id=task_group_activity_id,
+                    task_group_id=task_group_id,
+                    db=db,
+                )
+                if not success:
+                    return
+
+                # Log some info
+                logger.info("START")
+                for key, value in task_group.model_dump().items():
+                    logger.debug(f"task_group.{key}: {value}")
+
+                # Check that SSH connection works
+                try:
+                    fractal_ssh.check_connection()
+                except Exception as e:
+                    logger.error("Cannot establish SSH connection.")
+                    fail_and_cleanup(
+                        task_group=task_group,
+                        task_group_activity=activity,
+                        logger_name=LOGGER_NAME,
+                        log_file_path=log_file_path,
+                        exception=e,
+                        db=db,
+                    )
+                    return
+
+                try:
+
+                    # Check that the (remote) task_group path does not exist
+                    if fractal_ssh.remote_exists(task_group.path):
+                        error_msg = f"{task_group.path} already exists."
+                        logger.error(error_msg)
+                        fail_and_cleanup(
+                            task_group=task_group,
+                            task_group_activity=activity,
+                            logger_name=LOGGER_NAME,
+                            log_file_path=log_file_path,
+                            exception=FileExistsError(error_msg),
+                            db=db,
+                        )
+                        return
+
+                    # Create remote `task_group.path` and `script_dir_remote`
+                    # folders (note that because of `parents=True` we  are in
+                    # the `no error if existing, make parent directories as
+                    # needed` scenario for `mkdir`)
+                    script_dir_remote = (
+                        Path(task_group.path) / SCRIPTS_SUBFOLDER
+                    ).as_posix()
+                    fractal_ssh.mkdir(folder=task_group.path, parents=True)
+                    fractal_ssh.mkdir(folder=script_dir_remote, parents=True)
+
+                    # Write tar.gz file locally and send it to remote path,
+                    # and set task_group.archive_path
+                    tar_gz_filename = tar_gz_file.filename
+                    archive_path = (
+                        Path(task_group.path) / tar_gz_filename
+                    ).as_posix()
+                    tmp_archive_path = (
+                        Path(tmpdir) / tar_gz_filename
+                    ).as_posix()
+                    logger.info(
+                        f"Write tar.gz-file contents into {tmp_archive_path}"
+                    )
+                    with open(tmp_archive_path, "wb") as f:
+                        f.write(tar_gz_file.contents)
+                    fractal_ssh.send_file(
+                        local=tmp_archive_path,
+                        remote=archive_path,
+                    )
+                    task_group.archive_path = archive_path
+                    task_group = add_commit_refresh(obj=task_group, db=db)
+
+                    # Set `pixi_bin`
+                    settings = Inject(get_settings)
+                    pixi_home = settings.pixi.versions[task_group.pixi_version]
+
+                    replacements = {
+                        ("__PIXI_HOME__", pixi_home),
+                        ("__PACKAGE_DIR__", task_group.path),
+                        ("__TAR_GZ_PATH__", archive_path),
+                        (
+                            "__IMPORT_PACKAGE_NAME__",
+                            task_group.pkg_name.replace("-", "_"),
+                        ),
+                        ("__SOURCE_DIR_NAME__", SOURCE_DIR_NAME),
+                    }
+
+                    logger.info("installing - START")
+
+                    # Set status to ONGOING and refresh logs
+                    activity.status = TaskGroupActivityStatusV2.ONGOING
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    common_args = dict(
+                        script_dir_local=(
+                            Path(tmpdir) / SCRIPTS_SUBFOLDER
+                        ).as_posix(),
+                        script_dir_remote=script_dir_remote,
+                        prefix=(
+                            f"{int(time.time())}_"
+                            f"{TaskGroupActivityActionV2.COLLECT}"
+                        ),
+                        fractal_ssh=fractal_ssh,
+                        logger_name=LOGGER_NAME,
+                    )
+
+                    # Run the three pixi-related scripts
+                    _customize_and_run_template(
+                        template_filename="pixi_1_extract.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    _customize_and_run_template(
+                        template_filename="pixi_2_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    stdout = _customize_and_run_template(
+                        template_filename="pixi_3_post_install.sh",
+                        replacements=replacements,
+                        **common_args,
+                    )
+                    activity.log = get_current_log(log_file_path)
+                    activity = add_commit_refresh(obj=activity, db=db)
+
+                    # Parse stdout
+                    parsed_output = parse_collect_stdout(stdout)
+                    package_root_remote = parsed_output["package_root"]
+                    venv_size = parsed_output["venv_size"]
+                    venv_file_number = parsed_output["venv_file_number"]
+                    project_python_wrapper = parsed_output[
+                        "project_python_wrapper"
+                    ]
+
+                    # Read and validate remote manifest file
+                    manifest_path_remote = (
+                        f"{package_root_remote}/__FRACTAL_MANIFEST__.json"
+                    )
+                    pkg_manifest_dict = fractal_ssh.read_remote_json_file(
+                        manifest_path_remote
+                    )
+                    logger.info(f"Loaded {manifest_path_remote=}")
+                    pkg_manifest = ManifestV2(**pkg_manifest_dict)
+                    logger.info("Manifest is a valid ManifestV2")
+
+                    logger.info("_prepare_tasks_metadata - start")
+                    task_list = prepare_tasks_metadata(
+                        package_manifest=pkg_manifest,
+                        package_version=task_group.version,
+                        package_root=Path(package_root_remote),
+                        project_python_wrapper=Path(project_python_wrapper),
+                    )
+                    logger.info("_prepare_tasks_metadata - end")
+
+                    logger.info(
+                        "create_db_tasks_and_update_task_group - " "start"
+                    )
+                    create_db_tasks_and_update_task_group_sync(
+                        task_list=task_list,
+                        task_group_id=task_group.id,
+                        db=db,
+                    )
+                    logger.info("create_db_tasks_and_update_task_group - end")
+
+                    # NOTE: see issue 2626 about whether to keep `pixi.lock`
+                    # files in the database
+                    remote_pixi_lock_file = Path(
+                        task_group.path,
+                        SOURCE_DIR_NAME,
+                        "pixi.lock",
+                    ).as_posix()
+                    pixi_lock_contents = fractal_ssh.read_remote_text_file(
+                        remote_pixi_lock_file
+                    )
+
+                    # Update task_group data
+                    logger.info(
+                        "Add env_info, venv_size and venv_file_number "
+                        "to TaskGroupV2 - start"
+                    )
+                    task_group.env_info = pixi_lock_contents
+                    task_group.venv_size_in_kB = int(venv_size)
+                    task_group.venv_file_number = int(venv_file_number)
+                    task_group = add_commit_refresh(obj=task_group, db=db)
+                    logger.info(
+                        "Add env_info, venv_size and venv_file_number "
+                        "to TaskGroupV2 - end"
+                    )
+
+                    # Finalize (write metadata to DB)
+                    logger.info("finalising - START")
+                    activity.status = TaskGroupActivityStatusV2.OK
+                    activity.timestamp_ended = get_timestamp()
+                    activity = add_commit_refresh(obj=activity, db=db)
+                    logger.info("finalising - END")
+                    logger.info("END")
+                    reset_logger_handlers(logger)
+
+                except Exception as collection_e:
+                    # Delete corrupted package dir
+                    try:
+                        logger.info(
+                            f"Now delete remote folder {task_group.path}"
+                        )
+                        fractal_ssh.remove_folder(
+                            folder=task_group.path,
+                            safe_root=tasks_base_dir,
+                        )
+                        logger.info(
+                            f"Deleted remoted folder {task_group.path}"
+                        )
+                    except Exception as e_rm:
+                        logger.error(
+                            "Removing folder failed. "
+                            f"Original error:\n{str(e_rm)}"
+                        )
+                    fail_and_cleanup(
+                        task_group=task_group,
+                        task_group_activity=activity,
+                        log_file_path=log_file_path,
+                        logger_name=LOGGER_NAME,
+                        exception=collection_e,
+                        db=db,
+                    )