fractal-server 2.11.1__py3-none-any.whl → 2.12.0a0__py3-none-any.whl

fractal_server/tasks/v1/background_operations.py

@@ -1,352 +0,0 @@
-"""
-The main function exported from this module is `background_collect_pip`, which
-is used as a background task for the task-collection endpoint.
-"""
-import json
-from pathlib import Path
-from shutil import rmtree as shell_rmtree
-
-from ...string_tools import slugify_task_name_for_source_v1
-from ..utils import get_collection_log_v1
-from ..utils import get_collection_path
-from ..utils import get_log_path
-from ..v2.utils_package_names import normalize_package_name
-from ._TaskCollectPip import _TaskCollectPip
-from .utils import _init_venv_v1
-from fractal_server.app.db import DBSyncSession
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.models.v1 import State
-from fractal_server.app.models.v1 import Task
-from fractal_server.app.schemas.v1 import TaskCollectStatusV1
-from fractal_server.app.schemas.v1 import TaskCreateV1
-from fractal_server.app.schemas.v1 import TaskReadV1
-from fractal_server.logger import close_logger
-from fractal_server.logger import get_logger
-from fractal_server.logger import set_logger
-from fractal_server.utils import execute_command_async
-
-
-async def _pip_install(
-    venv_path: Path,
-    task_pkg: _TaskCollectPip,
-    logger_name: str,
-) -> Path:
-    """
-    Install package in venv
-
-    Args:
-        venv_path:
-        task_pkg:
-        logger_name:
-
-    Returns:
-        The location of the package.
-    """
-
-    logger = get_logger(logger_name)
-
-    pip = venv_path / "venv/bin/pip"
-
-    extras = f"[{task_pkg.package_extras}]" if task_pkg.package_extras else ""
-
-    if task_pkg.is_local_package:
-        pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
-    else:
-        version_string = (
-            f"=={task_pkg.package_version}" if task_pkg.package_version else ""
-        )
-        pip_install_str = f"{task_pkg.package}{extras}{version_string}"
-
-    cmd_install = f"{pip} install {pip_install_str}"
-    cmd_inspect = f"{pip} show {task_pkg.package}"
-
-    await execute_command_async(
-        cwd=venv_path,
-        command=f"{pip} install --upgrade pip",
-        logger_name=logger_name,
-    )
-    await execute_command_async(
-        cwd=venv_path, command=cmd_install, logger_name=logger_name
-    )
-    if task_pkg.pinned_package_versions:
-        for (
-            pinned_pkg_name,
-            pinned_pkg_version,
-        ) in task_pkg.pinned_package_versions.items():
-
-            logger.debug(
-                "Specific version required: "
-                f"{pinned_pkg_name}=={pinned_pkg_version}"
-            )
-            logger.debug(
-                "Preliminary check: verify that "
-                f"{pinned_pkg_version} is already installed"
-            )
-            stdout_inspect = await execute_command_async(
-                cwd=venv_path,
-                command=f"{pip} show {pinned_pkg_name}",
-                logger_name=logger_name,
-            )
-            current_version = next(
-                line.split()[-1]
-                for line in stdout_inspect.split("\n")
-                if line.startswith("Version:")
-            )
-            if current_version != pinned_pkg_version:
-                logger.debug(
-                    f"Currently installed version of {pinned_pkg_name} "
-                    f"({current_version}) differs from pinned version "
-                    f"({pinned_pkg_version}); "
-                    f"install version {pinned_pkg_version}."
-                )
-                await execute_command_async(
-                    cwd=venv_path,
-                    command=(
-                        f"{pip} install "
-                        f"{pinned_pkg_name}=={pinned_pkg_version}"
-                    ),
-                    logger_name=logger_name,
-                )
-            else:
-                logger.debug(
-                    f"Currently installed version of {pinned_pkg_name} "
-                    f"({current_version}) already matches the pinned version."
-                )
-
-    # Extract package installation path from `pip show`
-    stdout_inspect = await execute_command_async(
-        cwd=venv_path, command=cmd_inspect, logger_name=logger_name
-    )
-
-    location = Path(
-        next(
-            line.split()[-1]
-            for line in stdout_inspect.split("\n")
-            if line.startswith("Location:")
-        )
-    )
-
-    # NOTE
-    # https://packaging.python.org/en/latest/specifications/recording-installed-packages/
-    # This directory is named as {name}-{version}.dist-info, with name and
-    # version fields corresponding to Core metadata specifications. Both
-    # fields must be normalized (see the name normalization specification and
-    # the version normalization specification), and replace dash (-)
-    # characters with underscore (_) characters, so the .dist-info directory
-    # always has exactly one dash (-) character in its stem, separating the
-    # name and version fields.
-    package_root = location / (task_pkg.package.replace("-", "_"))
-    logger.debug(f"[_pip install] {location=}")
-    logger.debug(f"[_pip install] {task_pkg.package=}")
-    logger.debug(f"[_pip install] {package_root=}")
-    if not package_root.exists():
-        raise RuntimeError(
-            "Could not determine package installation location."
-        )
-    return package_root
-
-
-async def _create_venv_install_package(
-    *,
-    task_pkg: _TaskCollectPip,
-    path: Path,
-    logger_name: str,
-) -> tuple[Path, Path]:
-    """Create venv and install package
-
-    Args:
-        path: the directory in which to create the environment
-        task_pkg: object containing the different metadata required to install
-            the package
-
-    Returns:
-        python_bin: path to venv's python interpreter
-        package_root: the location of the package manifest
-    """
-
-    # Normalize package name
-    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
-    task_pkg.package = normalize_package_name(task_pkg.package)
-
-    python_bin = await _init_venv_v1(
-        path=path,
-        python_version=task_pkg.python_version,
-        logger_name=logger_name,
-    )
-    package_root = await _pip_install(
-        venv_path=path, task_pkg=task_pkg, logger_name=logger_name
-    )
-    return python_bin, package_root
-
-
-async def create_package_environment_pip(
-    *,
-    task_pkg: _TaskCollectPip,
-    venv_path: Path,
-    logger_name: str,
-) -> list[TaskCreateV1]:
-    """
-    Create environment, install package, and prepare task list
-    """
-
-    logger = get_logger(logger_name)
-
-    # Normalize package name
-    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
-    task_pkg.package = normalize_package_name(task_pkg.package)
-
-    # Only proceed if package, version and manifest attributes are set
-    task_pkg.check()
-
-    try:
-
-        logger.debug("Creating venv and installing package")
-        python_bin, package_root = await _create_venv_install_package(
-            path=venv_path,
-            task_pkg=task_pkg,
-            logger_name=logger_name,
-        )
-        logger.debug("Venv creation and package installation ended correctly.")
-
-        # Prepare task_list with appropriate metadata
-        logger.debug("Creating task list from manifest")
-        task_list = []
-        for t in task_pkg.package_manifest.task_list:
-            # Fill in attributes for TaskCreate
-            task_executable = package_root / t.executable
-            cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
-            task_name_slug = slugify_task_name_for_source_v1(t.name)
-            task_source = f"{task_pkg.package_source}:{task_name_slug}"
-            if not task_executable.exists():
-                raise FileNotFoundError(
-                    f"Cannot find executable `{task_executable}` "
-                    f"for task `{t.name}`"
-                )
-            manifest = task_pkg.package_manifest
-            if manifest.has_args_schemas:
-                additional_attrs = dict(
-                    args_schema_version=manifest.args_schema_version
-                )
-            else:
-                additional_attrs = {}
-            this_task = TaskCreateV1(
-                **t.dict(),
-                command=cmd,
-                version=task_pkg.package_version,
-                **additional_attrs,
-                source=task_source,
-            )
-            task_list.append(this_task)
-        logger.debug("Task list created correctly")
-    except Exception as e:
-        logger.error("Task manifest loading failed")
-        raise e
-    return task_list
-
-
-async def _insert_tasks(
-    task_list: list[TaskCreateV1],
-    db: DBSyncSession,
-) -> list[Task]:
-    """
-    Insert tasks into database
-    """
-    task_db_list = [Task(**t.dict()) for t in task_list]
-    db.add_all(task_db_list)
-    db.commit()
-    for t in task_db_list:
-        db.refresh(t)
-    db.close()
-    return task_db_list
-
-
-async def background_collect_pip(
-    state_id: int,
-    venv_path: Path,
-    task_pkg: _TaskCollectPip,
-) -> None:
-    """
-    Install package and collect tasks
-
-    Install a python package and collect the tasks it provides according to
-    the manifest.
-
-    In case of error, copy the log into the state and delete the package
-    directory.
-    """
-    logger_name = task_pkg.package.replace("/", "_")
-    logger = set_logger(
-        logger_name=logger_name,
-        log_file_path=get_log_path(venv_path),
-    )
-    logger.debug("Start background task collection")
-    for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
-        logger.debug(f"{key}: {value}")
-
-    with next(get_sync_db()) as db:
-        state: State = db.get(State, state_id)
-        data = TaskCollectStatusV1(**state.data)
-        data.info = None
-
-        try:
-            # install
-            logger.debug("Task-collection status: installing")
-            data.status = "installing"
-
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            task_list = await create_package_environment_pip(
-                venv_path=venv_path,
-                task_pkg=task_pkg,
-                logger_name=logger_name,
-            )
-
-            # collect
-            logger.debug("Task-collection status: collecting")
-            data.status = "collecting"
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            tasks = await _insert_tasks(task_list=task_list, db=db)
-
-            # finalise
-            logger.debug("Task-collection status: finalising")
-            collection_path = get_collection_path(venv_path)
-            data.task_list = [
-                TaskReadV1(**task.model_dump()) for task in tasks
-            ]
-            with collection_path.open("w") as f:
-                json.dump(data.sanitised_dict(), f, indent=2)
-
-            # Update DB
-            data.status = "OK"
-            data.log = get_collection_log_v1(venv_path)
-            state.data = data.sanitised_dict()
-            db.add(state)
-            db.merge(state)
-            db.commit()
-
-            # Write last logs to file
-            logger.debug("Task-collection status: OK")
-            logger.info("Background task collection completed successfully")
-            close_logger(logger)
-            db.close()
-
-        except Exception as e:
-            # Write last logs to file
-            logger.debug("Task-collection status: fail")
-            logger.info(f"Background collection failed. Original error: {e}")
-            close_logger(logger)
-
-            # Update db
-            data.status = "fail"
-            data.info = f"Original error: {e}"
-            data.log = get_collection_log_v1(venv_path)
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            db.close()
-
-            # Delete corrupted package dir
-            shell_rmtree(venv_path)

fractal_server/tasks/v1/endpoint_operations.py

@@ -1,156 +0,0 @@
-import json
-from pathlib import Path
-from typing import Optional
-from typing import Union
-from zipfile import ZipFile
-
-from ..v2.utils_package_names import normalize_package_name
-from ._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV1
-from .utils import get_python_interpreter_v1
-from fractal_server.app.schemas.v1 import ManifestV1
-from fractal_server.config import get_settings
-from fractal_server.logger import get_logger
-from fractal_server.syringe import Inject
-from fractal_server.utils import execute_command_async
-
-
-FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
-
-
-async def download_package(
-    *,
-    task_pkg: _TaskCollectPipV1,
-    dest: Union[str, Path],
-) -> Path:
-    """
-    Download package to destination
-    """
-    interpreter = get_python_interpreter_v1(version=task_pkg.python_version)
-    pip = f"{interpreter} -m pip"
-    version = (
-        f"=={task_pkg.package_version}" if task_pkg.package_version else ""
-    )
-    package_and_version = f"{task_pkg.package}{version}"
-    cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
-    stdout = await execute_command_async(command=cmd, cwd=Path("."))
-    pkg_file = next(
-        line.split()[-1] for line in stdout.split("\n") if "Saved" in line
-    )
-    return Path(pkg_file)
-
-
-def _load_manifest_from_wheel(
-    path: Path, wheel: ZipFile, logger_name: Optional[str] = None
-) -> ManifestV1:
-    logger = get_logger(logger_name)
-    namelist = wheel.namelist()
-    try:
-        manifest = next(
-            name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
-        )
-    except StopIteration:
-        msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
-        logger.error(msg)
-        raise ValueError(msg)
-    with wheel.open(manifest) as manifest_fd:
-        manifest_dict = json.load(manifest_fd)
-    manifest_version = str(manifest_dict["manifest_version"])
-    if manifest_version == "1":
-        pkg_manifest = ManifestV1(**manifest_dict)
-        return pkg_manifest
-    else:
-        msg = f"Manifest version {manifest_version=} not supported"
-        logger.error(msg)
-        raise ValueError(msg)
-
-
-def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
-    """
-    Inspect task package to extract version, name and manifest
-
-    Note that this only works with wheel files, which have a well-defined
-    dist-info section. If we need to generalize to to tar.gz archives, we would
-    need to go and look for `PKG-INFO`.
-
-    Note: package name is normalized via `_normalize_package_name`.
-
-    Args:
-        path: Path
-            the path in which the package is saved
-
-    Returns:
-        version_manifest: A dictionary containing `version`, the version of the
-        pacakge, and `manifest`, the Fractal manifest object relative to the
-        tasks.
-    """
-
-    logger = get_logger(logger_name)
-
-    if not path.as_posix().endswith(".whl"):
-        raise ValueError(
-            f"Only wheel packages are supported, given {path.as_posix()}."
-        )
-
-    with ZipFile(path) as wheel:
-        namelist = wheel.namelist()
-
-        # Read and validate task manifest
-        logger.debug(f"Now reading manifest for {path.as_posix()}")
-        pkg_manifest = _load_manifest_from_wheel(
-            path, wheel, logger_name=logger_name
-        )
-        logger.debug("Manifest read correctly.")
-
-        # Read package name and version from *.dist-info/METADATA
-        logger.debug(
-            f"Now reading package name and version for {path.as_posix()}"
-        )
-        metadata = next(
-            name for name in namelist if "dist-info/METADATA" in name
-        )
-        with wheel.open(metadata) as metadata_fd:
-            meta = metadata_fd.read().decode("utf-8")
-            pkg_name = next(
-                line.split()[-1]
-                for line in meta.splitlines()
-                if line.startswith("Name")
-            )
-            pkg_version = next(
-                line.split()[-1]
-                for line in meta.splitlines()
-                if line.startswith("Version")
-            )
-        logger.debug("Package name and version read correctly.")
-
-    # Normalize package name:
-    pkg_name = normalize_package_name(pkg_name)
-
-    info = dict(
-        pkg_name=pkg_name,
-        pkg_version=pkg_version,
-        pkg_manifest=pkg_manifest,
-    )
-    return info
-
-
-def create_package_dir_pip(
-    *,
-    task_pkg: _TaskCollectPipV1,
-    create: bool = True,
-) -> Path:
-    """
-    Create venv folder for a task package and return corresponding Path object
-    """
-    settings = Inject(get_settings)
-    user = FRACTAL_PUBLIC_TASK_SUBDIR
-    if task_pkg.package_version is None:
-        raise ValueError(
-            f"Cannot create venv folder for package `{task_pkg.package}` "
-            "with `version=None`."
-        )
-    normalized_package = normalize_package_name(task_pkg.package)
-    package_dir = f"{normalized_package}{task_pkg.package_version}"
-    venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
-    if create:
-        venv_path.mkdir(exist_ok=False, parents=True)
-    return venv_path

fractal_server/tasks/v1/get_collection_data.py

@@ -1,14 +0,0 @@
-import json
-from pathlib import Path
-
-from fractal_server.app.schemas.v1 import TaskCollectStatusV1
-from fractal_server.tasks.utils import get_absolute_venv_path_v1
-from fractal_server.tasks.utils import get_collection_path
-
-
-def get_collection_data(venv_path: Path) -> TaskCollectStatusV1:
-    package_path = get_absolute_venv_path_v1(venv_path)
-    collection_path = get_collection_path(package_path)
-    with collection_path.open() as f:
-        data = json.load(f)
-    return TaskCollectStatusV1(**data)

fractal_server/tasks/v1/utils.py

@@ -1,67 +0,0 @@
-from pathlib import Path
-from typing import Optional
-
-from fractal_server.logger import get_logger
-from fractal_server.utils import execute_command_async
-
-
-def get_python_interpreter_v1(version: Optional[str] = None) -> str:
-    """
-    Return the path to the python interpreter
-
-    Args:
-        version: Python version
-
-    Raises:
-        ValueError: If the python version requested is not available on the
-                    host.
-
-    Returns:
-        interpreter: string representing the python executable or its path
-    """
-    import shutil
-    import sys
-
-    if version:
-        interpreter = shutil.which(f"python{version}")
-        if not interpreter:
-            raise ValueError(
-                f"Python version {version} not available on host."
-            )
-    else:
-        interpreter = sys.executable
-
-    return interpreter
-
-
-async def _init_venv_v1(
-    *,
-    path: Path,
-    python_version: Optional[str] = None,
-    logger_name: str,
-) -> Path:
-    """
-    Set a virtual environment at `path/venv`
-
-    Args:
-        path : Path
-            path to directory in which to set up the virtual environment
-        python_version : default=None
-            Python version the virtual environment will be based upon
-
-    Returns:
-        python_bin : Path
-            path to python interpreter
-    """
-    logger = get_logger(logger_name)
-    logger.debug(f"[_init_venv] {path=}")
-    interpreter = get_python_interpreter_v1(version=python_version)
-    logger.debug(f"[_init_venv] {interpreter=}")
-    await execute_command_async(
-        cwd=path,
-        command=f"{interpreter} -m venv venv",
-        logger_name=logger_name,
-    )
-    python_bin = path / "venv/bin/python"
-    logger.debug(f"[_init_venv] {python_bin=}")
-    return python_bin