fractal-server 1.4.3a0__py3-none-any.whl → 1.4.3a1__py3-none-any.whl

fractal_server/tasks/background_operations.py

@@ -0,0 +1,384 @@
+"""
+The main function exported from this module is `background_collect_pip`, which
+is used as a background task for the task-collection endpoint.
+"""
+import json
+from pathlib import Path
+from shutil import rmtree as shell_rmtree
+from typing import Optional
+
+from fractal_server.app.db import DBSyncSession
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import State
+from fractal_server.app.models import Task
+from fractal_server.app.schemas import TaskCollectStatus
+from fractal_server.app.schemas import TaskCreate
+from fractal_server.app.schemas import TaskRead
+from fractal_server.logger import close_logger
+from fractal_server.logger import get_logger
+from fractal_server.logger import set_logger
+from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
+from fractal_server.tasks.utils import _normalize_package_name
+from fractal_server.tasks.utils import get_collection_log
+from fractal_server.tasks.utils import get_collection_path
+from fractal_server.tasks.utils import get_log_path
+from fractal_server.tasks.utils import get_python_interpreter
+from fractal_server.tasks.utils import slugify_task_name
+from fractal_server.utils import execute_command
+
+
+async def _init_venv(
+    *,
+    path: Path,
+    python_version: Optional[str] = None,
+    logger_name: str,
+) -> Path:
+    """
+    Set a virtual environment at `path/venv`
+
+    Args:
+        path : Path
+            path to directory in which to set up the virtual environment
+        python_version : default=None
+            Python version the virtual environment will be based upon
+
+    Returns:
+        python_bin : Path
+            path to python interpreter
+    """
+    logger = get_logger(logger_name)
+    logger.debug(f"[_init_venv] {path=}")
+    interpreter = get_python_interpreter(version=python_version)
+    logger.debug(f"[_init_venv] {interpreter=}")
+    await execute_command(
+        cwd=path,
+        command=f"{interpreter} -m venv venv",
+        logger_name=logger_name,
+    )
+    python_bin = path / "venv/bin/python"
+    logger.debug(f"[_init_venv] {python_bin=}")
+    return python_bin
+
+
+async def _pip_install(
+    venv_path: Path,
+    task_pkg: _TaskCollectPip,
+    logger_name: str,
+) -> Path:
+    """
+    Install package in venv
+
+    Args:
+        venv_path:
+        task_pkg:
+        logger_name:
+
+    Returns:
+        The location of the package.
+    """
+
+    logger = get_logger(logger_name)
+
+    pip = venv_path / "venv/bin/pip"
+
+    extras = f"[{task_pkg.package_extras}]" if task_pkg.package_extras else ""
+
+    if task_pkg.is_local_package:
+        pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
+    else:
+        version_string = (
+            f"=={task_pkg.package_version}" if task_pkg.package_version else ""
+        )
+        pip_install_str = f"{task_pkg.package}{extras}{version_string}"
+
+    cmd_install = f"{pip} install {pip_install_str}"
+    cmd_inspect = f"{pip} show {task_pkg.package}"
+
+    await execute_command(
+        cwd=venv_path,
+        command=f"{pip} install --upgrade pip",
+        logger_name=logger_name,
+    )
+    await execute_command(
+        cwd=venv_path, command=cmd_install, logger_name=logger_name
+    )
+    if task_pkg.pinned_package_versions:
+        for (
+            pinned_pkg_name,
+            pinned_pkg_version,
+        ) in task_pkg.pinned_package_versions.items():
+
+            logger.debug(
+                "Specific version required: "
+                f"{pinned_pkg_name}=={pinned_pkg_version}"
+            )
+            logger.debug(
+                "Preliminary check: verify that "
+                f"{pinned_pkg_version} is already installed"
+            )
+            stdout_inspect = await execute_command(
+                cwd=venv_path,
+                command=f"{pip} show {pinned_pkg_name}",
+                logger_name=logger_name,
+            )
+            current_version = next(
+                line.split()[-1]
+                for line in stdout_inspect.split("\n")
+                if line.startswith("Version:")
+            )
+            if current_version != pinned_pkg_version:
+                logger.debug(
+                    f"Currently installed version of {pinned_pkg_name} "
+                    f"({current_version}) differs from pinned version "
+                    f"({pinned_pkg_version}); "
+                    f"install version {pinned_pkg_version}."
+                )
+                await execute_command(
+                    cwd=venv_path,
+                    command=(
+                        f"{pip} install "
+                        f"{pinned_pkg_name}=={pinned_pkg_version}"
+                    ),
+                    logger_name=logger_name,
+                )
+            else:
+                logger.debug(
+                    f"Currently installed version of {pinned_pkg_name} "
+                    f"({current_version}) already matches the pinned version."
+                )
+
+    # Extract package installation path from `pip show`
+    stdout_inspect = await execute_command(
+        cwd=venv_path, command=cmd_inspect, logger_name=logger_name
+    )
+
+    location = Path(
+        next(
+            line.split()[-1]
+            for line in stdout_inspect.split("\n")
+            if line.startswith("Location:")
+        )
+    )
+
+    # NOTE
+    # https://packaging.python.org/en/latest/specifications/recording-installed-packages/
+    # This directory is named as {name}-{version}.dist-info, with name and
+    # version fields corresponding to Core metadata specifications. Both
+    # fields must be normalized (see the name normalization specification and
+    # the version normalization specification), and replace dash (-)
+    # characters with underscore (_) characters, so the .dist-info directory
+    # always has exactly one dash (-) character in its stem, separating the
+    # name and version fields.
+    package_root = location / (task_pkg.package.replace("-", "_"))
+    logger.debug(f"[_pip install] {location=}")
+    logger.debug(f"[_pip install] {task_pkg.package=}")
+    logger.debug(f"[_pip install] {package_root=}")
+    if not package_root.exists():
+        raise RuntimeError(
+            "Could not determine package installation location."
+        )
+    return package_root
+
+
+async def _create_venv_install_package(
+    *,
+    task_pkg: _TaskCollectPip,
+    path: Path,
+    logger_name: str,
+) -> tuple[Path, Path]:
+    """Create venv and install package
+
+    Args:
+        path: the directory in which to create the environment
+        task_pkg: object containing the different metadata required to install
+            the package
+
+    Returns:
+        python_bin: path to venv's python interpreter
+        package_root: the location of the package manifest
+    """
+
+    # Normalize package name
+    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
+    task_pkg.package = _normalize_package_name(task_pkg.package)
+
+    python_bin = await _init_venv(
+        path=path,
+        python_version=task_pkg.python_version,
+        logger_name=logger_name,
+    )
+    package_root = await _pip_install(
+        venv_path=path, task_pkg=task_pkg, logger_name=logger_name
+    )
+    return python_bin, package_root
+
+
+async def create_package_environment_pip(
+    *,
+    task_pkg: _TaskCollectPip,
+    venv_path: Path,
+    logger_name: str,
+) -> list[TaskCreate]:
+    """
+    Create environment, install package, and prepare task list
+    """
+
+    logger = get_logger(logger_name)
+
+    # Normalize package name
+    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
+    task_pkg.package = _normalize_package_name(task_pkg.package)
+
+    # Only proceed if package, version and manifest attributes are set
+    task_pkg.check()
+
+    try:
+
+        logger.debug("Creating venv and installing package")
+        python_bin, package_root = await _create_venv_install_package(
+            path=venv_path,
+            task_pkg=task_pkg,
+            logger_name=logger_name,
+        )
+        logger.debug("Venv creation and package installation ended correctly.")
+
+        # Prepare task_list with appropriate metadata
+        logger.debug("Creating task list from manifest")
+        task_list = []
+        for t in task_pkg.package_manifest.task_list:
+            # Fill in attributes for TaskCreate
+            task_executable = package_root / t.executable
+            cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
+            task_name_slug = slugify_task_name(t.name)
+            task_source = f"{task_pkg.package_source}:{task_name_slug}"
+            if not task_executable.exists():
+                raise FileNotFoundError(
+                    f"Cannot find executable `{task_executable}` "
+                    f"for task `{t.name}`"
+                )
+            manifest = task_pkg.package_manifest
+            if manifest.has_args_schemas:
+                additional_attrs = dict(
+                    args_schema_version=manifest.args_schema_version
+                )
+            else:
+                additional_attrs = {}
+            this_task = TaskCreate(
+                **t.dict(),
+                command=cmd,
+                version=task_pkg.package_version,
+                **additional_attrs,
+                source=task_source,
+            )
+            task_list.append(this_task)
+        logger.debug("Task list created correctly")
+    except Exception as e:
+        logger.error("Task manifest loading failed")
+        raise e
+    return task_list
+
+
+async def _insert_tasks(
+    task_list: list[TaskCreate],
+    db: DBSyncSession,
+) -> list[Task]:
+    """
+    Insert tasks into database
+    """
+    task_db_list = [Task(**t.dict()) for t in task_list]
+    db.add_all(task_db_list)
+    db.commit()
+    for t in task_db_list:
+        db.refresh(t)
+    db.close()
+    return task_db_list
+
+
+async def background_collect_pip(
+    state_id: int,
+    venv_path: Path,
+    task_pkg: _TaskCollectPip,
+) -> None:
+    """
+    Install package and collect tasks
+
+    Install a python package and collect the tasks it provides according to
+    the manifest.
+
+    In case of error, copy the log into the state and delete the package
+    directory.
+    """
+    logger_name = task_pkg.package.replace("/", "_")
+    logger = set_logger(
+        logger_name=logger_name,
+        log_file_path=get_log_path(venv_path),
+    )
+    logger.debug("Start background task collection")
+    for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
+        logger.debug(f"{key}: {value}")
+
+    with next(get_sync_db()) as db:
+        state: State = db.get(State, state_id)
+        data = TaskCollectStatus(**state.data)
+        data.info = None
+
+        try:
+            # install
+            logger.debug("Task-collection status: installing")
+            data.status = "installing"
+
+            state.data = data.sanitised_dict()
+            db.merge(state)
+            db.commit()
+            task_list = await create_package_environment_pip(
+                venv_path=venv_path,
+                task_pkg=task_pkg,
+                logger_name=logger_name,
+            )
+
+            # collect
+            logger.debug("Task-collection status: collecting")
+            data.status = "collecting"
+            state.data = data.sanitised_dict()
+            db.merge(state)
+            db.commit()
+            tasks = await _insert_tasks(task_list=task_list, db=db)
+
+            # finalise
+            logger.debug("Task-collection status: finalising")
+            collection_path = get_collection_path(venv_path)
+            data.task_list = [TaskRead(**task.model_dump()) for task in tasks]
+            with collection_path.open("w") as f:
+                json.dump(data.sanitised_dict(), f)
+
+            # Update DB
+            data.status = "OK"
+            data.log = get_collection_log(venv_path)
+            state.data = data.sanitised_dict()
+            db.add(state)
+            db.merge(state)
+            db.commit()
+
+            # Write last logs to file
+            logger.debug("Task-collection status: OK")
+            logger.info("Background task collection completed successfully")
+            close_logger(logger)
+            db.close()
+
+        except Exception as e:
+            # Write last logs to file
+            logger.debug("Task-collection status: fail")
+            logger.info(f"Background collection failed. Original error: {e}")
+            close_logger(logger)
+
+            # Update db
+            data.status = "fail"
+            data.info = f"Original error: {e}"
+            data.log = get_collection_log(venv_path)
+            state.data = data.sanitised_dict()
+            db.merge(state)
+            db.commit()
+            db.close()
+
+            # Delete corrupted package dir
+            shell_rmtree(venv_path)

fractal_server/tasks/endpoint_operations.py

@@ -0,0 +1,167 @@
+import json
+from pathlib import Path
+from typing import Optional
+from typing import Union
+from zipfile import ZipFile
+
+from fractal_server.app.schemas import ManifestV1
+from fractal_server.app.schemas import TaskCollectStatus
+from fractal_server.config import get_settings
+from fractal_server.logger import get_logger
+from fractal_server.syringe import Inject
+from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
+from fractal_server.tasks.utils import _normalize_package_name
+from fractal_server.tasks.utils import get_absolute_venv_path
+from fractal_server.tasks.utils import get_collection_path
+from fractal_server.tasks.utils import get_python_interpreter
+from fractal_server.utils import execute_command
+
+
+FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
+
+
+def get_collection_data(venv_path: Path) -> TaskCollectStatus:
+    package_path = get_absolute_venv_path(venv_path)
+    collection_path = get_collection_path(package_path)
+    with collection_path.open() as f:
+        data = json.load(f)
+    return TaskCollectStatus(**data)
+
+
+async def download_package(
+    *,
+    task_pkg: _TaskCollectPip,
+    dest: Union[str, Path],
+):
+    """
+    Download package to destination
+    """
+    interpreter = get_python_interpreter(version=task_pkg.python_version)
+    pip = f"{interpreter} -m pip"
+    version = (
+        f"=={task_pkg.package_version}" if task_pkg.package_version else ""
+    )
+    package_and_version = f"{task_pkg.package}{version}"
+    cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
+    stdout = await execute_command(command=cmd, cwd=Path("."))
+    pkg_file = next(
+        line.split()[-1] for line in stdout.split("\n") if "Saved" in line
+    )
+    return Path(pkg_file)
+
+
+def _load_manifest_from_wheel(
+    path: Path, wheel: ZipFile, logger_name: Optional[str] = None
+) -> ManifestV1:
+    logger = get_logger(logger_name)
+    namelist = wheel.namelist()
+    try:
+        manifest = next(
+            name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
+        )
+    except StopIteration:
+        msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
+        logger.error(msg)
+        raise ValueError(msg)
+    with wheel.open(manifest) as manifest_fd:
+        manifest_dict = json.load(manifest_fd)
+    manifest_version = str(manifest_dict["manifest_version"])
+    if manifest_version == "1":
+        pkg_manifest = ManifestV1(**manifest_dict)
+        return pkg_manifest
+    else:
+        msg = f"Manifest version {manifest_version=} not supported"
+        logger.error(msg)
+        raise ValueError(msg)
+
+
+def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
+    """
+    Inspect task package to extract version, name and manifest
+
+    Note that this only works with wheel files, which have a well-defined
+    dist-info section. If we need to generalize to to tar.gz archives, we would
+    need to go and look for `PKG-INFO`.
+
+    Note: package name is normalized via `_normalize_package_name`.
+
+    Args:
+        path: Path
+            the path in which the package is saved
+
+    Returns:
+        version_manifest: A dictionary containing `version`, the version of the
+        pacakge, and `manifest`, the Fractal manifest object relative to the
+        tasks.
+    """
+
+    logger = get_logger(logger_name)
+
+    if not path.as_posix().endswith(".whl"):
+        raise ValueError(
+            f"Only wheel packages are supported, given {path.as_posix()}."
+        )
+
+    with ZipFile(path) as wheel:
+        namelist = wheel.namelist()
+
+        # Read and validate task manifest
+        logger.debug(f"Now reading manifest for {path.as_posix()}")
+        pkg_manifest = _load_manifest_from_wheel(
+            path, wheel, logger_name=logger_name
+        )
+        logger.debug("Manifest read correctly.")
+
+        # Read package name and version from *.dist-info/METADATA
+        logger.debug(
+            f"Now reading package name and version for {path.as_posix()}"
+        )
+        metadata = next(
+            name for name in namelist if "dist-info/METADATA" in name
+        )
+        with wheel.open(metadata) as metadata_fd:
+            meta = metadata_fd.read().decode("utf-8")
+            pkg_name = next(
+                line.split()[-1]
+                for line in meta.splitlines()
+                if line.startswith("Name")
+            )
+            pkg_version = next(
+                line.split()[-1]
+                for line in meta.splitlines()
+                if line.startswith("Version")
+            )
+        logger.debug("Package name and version read correctly.")
+
+    # Normalize package name:
+    pkg_name = _normalize_package_name(pkg_name)
+
+    info = dict(
+        pkg_name=pkg_name,
+        pkg_version=pkg_version,
+        pkg_manifest=pkg_manifest,
+    )
+    return info
+
+
+def create_package_dir_pip(
+    *,
+    task_pkg: _TaskCollectPip,
+    create: bool = True,
+) -> Path:
+    """
+    Create venv folder for a task package and return corresponding Path object
+    """
+    settings = Inject(get_settings)
+    user = FRACTAL_PUBLIC_TASK_SUBDIR
+    if task_pkg.package_version is None:
+        raise ValueError(
+            f"Cannot create venv folder for package `{task_pkg.package}` "
+            "with `version=None`."
+        )
+    normalized_package = _normalize_package_name(task_pkg.package)
+    package_dir = f"{normalized_package}{task_pkg.package_version}"
+    venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
+    if create:
+        venv_path.mkdir(exist_ok=False, parents=True)
+    return venv_path

fractal_server/tasks/utils.py

@@ -0,0 +1,86 @@
+import re
+import shutil
+import sys
+from pathlib import Path
+from typing import Optional
+
+from fractal_server.config import get_settings
+from fractal_server.syringe import Inject
+
+COLLECTION_FILENAME = "collection.json"
+COLLECTION_LOG_FILENAME = "collection.log"
+
+
+def get_python_interpreter(version: Optional[str] = None) -> str:
+    """
+    Return the path to the python interpreter
+
+    Args:
+        version: Python version
+
+    Raises:
+        ValueError: If the python version requested is not available on the
+                    host.
+
+    Returns:
+        interpreter: string representing the python executable or its path
+    """
+    if version:
+        interpreter = shutil.which(f"python{version}")
+        if not interpreter:
+            raise ValueError(
+                f"Python version {version} not available on host."
+            )
+    else:
+        interpreter = sys.executable
+
+    return interpreter
+
+
+def slugify_task_name(task_name: str) -> str:
+    return task_name.replace(" ", "_").lower()
+
+
+def get_absolute_venv_path(venv_path: Path) -> Path:
+    """
+    If a path is not absolute, make it a relative path of FRACTAL_TASKS_DIR.
+    """
+    if venv_path.is_absolute():
+        package_path = venv_path
+    else:
+        settings = Inject(get_settings)
+        package_path = settings.FRACTAL_TASKS_DIR / venv_path
+    return package_path
+
+
+def get_collection_path(base: Path) -> Path:
+    return base / COLLECTION_FILENAME
+
+
+def get_log_path(base: Path) -> Path:
+    return base / COLLECTION_LOG_FILENAME
+
+
+def get_collection_log(venv_path: Path) -> str:
+    package_path = get_absolute_venv_path(venv_path)
+    log_path = get_log_path(package_path)
+    log = log_path.open().read()
+    return log
+
+
+def _normalize_package_name(name: str) -> str:
+    """
+    Implement PyPa specifications for package-name normalization
+
+    The name should be lowercased with all runs of the characters `.`, `-`,
+    or `_` replaced with a single `-` character. This can be implemented in
+    Python with the re module.
+    (https://packaging.python.org/en/latest/specifications/name-normalization)
+
+    Args:
+        name: The non-normalized package name.
+
+    Returns:
+        The normalized package name.
+    """
+    return re.sub(r"[-_.]+", "-", name).lower()

{fractal_server-1.4.3a0.dist-info → fractal_server-1.4.3a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 1.4.3a0
+Version: 1.4.3a1
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause