fractal-server 2.11.0a10__py3-none-any.whl → 2.12.0a0__py3-none-any.whl

fractal_server/data_migrations/2_11_0.py

@@ -1,168 +0,0 @@
-import logging
-from typing import Union
-
-from sqlalchemy.orm.attributes import flag_modified
-from sqlmodel import select
-
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.models import DatasetV2
-from fractal_server.app.models import JobV2
-from fractal_server.app.models import ProjectV2
-from fractal_server.app.models import WorkflowTaskV2
-from fractal_server.app.models import WorkflowV2
-from fractal_server.app.schemas.v2 import DatasetReadV2
-from fractal_server.app.schemas.v2 import JobReadV2
-from fractal_server.app.schemas.v2 import ProjectReadV2
-from fractal_server.app.schemas.v2 import TaskReadV2
-from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
-from fractal_server.images.models import AttributeFiltersType
-
-logger = logging.getLogger("fix_db")
-logger.setLevel(logging.INFO)
-
-
-def dict_values_to_list(
-    input_dict: dict[str, Union[int, float, bool, str, None]],
-    identifier: str,
-) -> tuple[AttributeFiltersType, bool]:
-    was_there_a_warning = False
-    for k, v in input_dict.items():
-        if not isinstance(v, (int, float, bool, str, type(None))):
-            error_msg = (
-                f"Attribute '{k}' from '{identifier}' "
-                "has invalid type '{type(v)}'."
-            )
-            logger.error(error_msg)
-            raise RuntimeError(error_msg)
-        elif v is None:
-            logger.warning(
-                f"Attribute '{k}' from '{identifier}' is "
-                "None and it will be removed."
-            )
-            was_there_a_warning = True
-        else:
-            input_dict[k] = [v]
-    return input_dict, was_there_a_warning
-
-
-def fix_db():
-    logger.info("START execution of fix_db function")
-
-    with next(get_sync_db()) as db:
-        # DatasetV2.filters
-        stm = select(DatasetV2).order_by(DatasetV2.id)
-        datasets = db.execute(stm).scalars().all()
-        for ds in datasets:
-            logger.info(f"DatasetV2[{ds.id}] START")
-            if ds.filters is None:
-                logger.info(f"DatasetV2[{ds.id}] SKIP")
-                continue
-
-            ds.attribute_filters, warning = dict_values_to_list(
-                ds.filters["attributes"],
-                f"Dataset[{ds.id}].filters.attributes",
-            )
-            if warning:
-                proj = db.get(ProjectV2, ds.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{ds.id=}, "
-                    f"{ds.name=}"
-                )
-            ds.type_filters = ds.filters["types"]
-            ds.filters = None
-            for i, h in enumerate(ds.history):
-                ds.history[i]["workflowtask"]["type_filters"] = h[
-                    "workflowtask"
-                ]["input_filters"]["types"]
-                ds.history[i]["workflowtask"].pop("input_filters")
-            flag_modified(ds, "history")
-            DatasetReadV2(
-                **ds.model_dump(),
-                project=ProjectReadV2(**ds.project.model_dump()),
-            )
-            db.add(ds)
-            logger.info(f"DatasetV2[{ds.id}] END - fixed filters")
-
-        logger.info("------ switch from dataset to workflowtasks ------")
-
-        # WorkflowTaskV2.input_filters
-        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
-        wftasks = db.execute(stm).scalars().all()
-        for wft in wftasks:
-            logger.info(f"WorkflowTaskV2[{wft.id}] START")
-            if wft.input_filters is None:
-                logger.info(f"WorkflowTaskV2[{wft.id}] SKIP")
-                continue
-            wft.type_filters = wft.input_filters["types"]
-            if wft.input_filters["attributes"]:
-                logger.warning(
-                    "Removing input_filters['attributes']. "
-                    f"(previous value: {wft.input_filters['attributes']})"
-                )
-                wf = db.get(WorkflowV2, wft.workflow_id)
-                proj = db.get(ProjectV2, wf.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{wf.id=}, "
-                    f"{wf.name=}, "
-                    f"{wft.task.name=}"
-                )
-            wft.input_filters = None
-            flag_modified(wft, "input_filters")
-            WorkflowTaskReadV2(
-                **wft.model_dump(),
-                task=TaskReadV2(**wft.task.model_dump()),
-            )
-            db.add(wft)
-            logger.info(f"WorkflowTaskV2[{wft.id}] END - fixed filters")
-
-        logger.info("------ switch from workflowtasks to jobs ------")
-
-        # JOBS V2
-        stm = select(JobV2).order_by(JobV2.id)
-        jobs = db.execute(stm).scalars().all()
-        for job in jobs:
-            logger.info(f"JobV2[{job.id}] START")
-            if "filters" not in job.dataset_dump.keys():
-                logger.info(f"JobV2[{job.id}] SKIP")
-                continue
-            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
-                "types"
-            ]
-            (
-                job.dataset_dump["attribute_filters"],
-                warning,
-            ) = dict_values_to_list(
-                job.dataset_dump["filters"]["attributes"],
-                f"JobV2[{job.id}].dataset_dump.filters.attributes",
-            )
-            if warning and job.project_id is not None:
-                proj = db.get(ProjectV2, job.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{job.id=}, "
-                    f"{job.start_timestamp=}, "
-                    f"{job.end_timestamp=}, "
-                    f"{job.dataset_id=}, "
-                    f"{job.workflow_id=}."
-                )
-            job.dataset_dump.pop("filters")
-            flag_modified(job, "dataset_dump")
-            JobReadV2(**job.model_dump())
-            db.add(job)
-            logger.info(f"JobV2[{job.id}] END - fixed filters")
-
-        db.commit()
-        logger.info("Changes committed.")
-
-    logger.info("END execution of fix_db function")

fractal_server/tasks/v1/_TaskCollectPip.py

@@ -1,103 +0,0 @@
-from pathlib import Path
-from typing import Optional
-
-from pydantic import root_validator
-
-from fractal_server.app.schemas.v1 import ManifestV1
-from fractal_server.app.schemas.v1 import TaskCollectPipV1
-
-
-class _TaskCollectPip(TaskCollectPipV1):
-    """
-    Internal TaskCollectPip schema
-
-    Differences with its parent class (`TaskCollectPip`):
-
-        1. We check if the package corresponds to a path in the filesystem, and
-           whether it exists (via new validator `check_local_package`, new
-           method `is_local_package` and new attribute `package_path`).
-        2. We include an additional `package_manifest` attribute.
-        3. We expose an additional attribute `package_name`, which is filled
-           during task collection.
-    """
-
-    package_name: Optional[str] = None
-    package_path: Optional[Path] = None
-    package_manifest: Optional[ManifestV1] = None
-
-    @property
-    def is_local_package(self) -> bool:
-        return bool(self.package_path)
-
-    @root_validator(pre=True)
-    def check_local_package(cls, values):
-        """
-        Checks if package corresponds to an existing path on the filesystem
-
-        In this case, the user is providing directly a package file, rather
-        than a remote one from PyPI. We set the `package_path` attribute and
-        get the actual package name and version from the package file name.
-        """
-        if "/" in values["package"]:
-            package_path = Path(values["package"])
-            if not package_path.is_absolute():
-                raise ValueError("Package path must be absolute")
-            if package_path.exists():
-                values["package_path"] = package_path
-                (
-                    values["package"],
-                    values["version"],
-                    *_,
-                ) = package_path.name.split("-")
-            else:
-                raise ValueError(f"Package {package_path} does not exist.")
-        return values
-
-    @property
-    def package_source(self) -> str:
-        """
-        NOTE: As of PR #1188 in `fractal-server`, the attribute
-        `self.package_name` is normalized; this means e.g. that `_` is
-        replaced by `-`. To guarantee backwards compatibility with
-        `Task.source` attributes created before this change, we still replace
-        `-` with `_` upon generation of the `source` attribute, in this
-        method.
-        """
-        if not self.package_name or not self.package_version:
-            raise ValueError(
-                "Cannot construct `package_source` property with "
-                f"{self.package_name=} and {self.package_version=}."
-            )
-        if self.is_local_package:
-            collection_type = "pip_local"
-        else:
-            collection_type = "pip_remote"
-
-        package_extras = self.package_extras or ""
-        if self.python_version:
-            python_version = f"py{self.python_version}"
-        else:
-            python_version = ""
-
-        source = ":".join(
-            (
-                collection_type,
-                self.package_name.replace("-", "_"),  # see method docstring
-                self.package_version,
-                package_extras,
-                python_version,
-            )
-        )
-        return source
-
-    def check(self):
-        """
-        Verify that the package has all attributes that are needed to continue
-        with task collection
-        """
-        if not self.package_name:
-            raise ValueError("`package_name` attribute is not set")
-        if not self.package_version:
-            raise ValueError("`package_version` attribute is not set")
-        if not self.package_manifest:
-            raise ValueError("`package_manifest` attribute is not set")

fractal_server/tasks/v1/background_operations.py

@@ -1,352 +0,0 @@
-"""
-The main function exported from this module is `background_collect_pip`, which
-is used as a background task for the task-collection endpoint.
-"""
-import json
-from pathlib import Path
-from shutil import rmtree as shell_rmtree
-
-from ...string_tools import slugify_task_name_for_source_v1
-from ..utils import get_collection_log_v1
-from ..utils import get_collection_path
-from ..utils import get_log_path
-from ..v2.utils_package_names import normalize_package_name
-from ._TaskCollectPip import _TaskCollectPip
-from .utils import _init_venv_v1
-from fractal_server.app.db import DBSyncSession
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.models.v1 import State
-from fractal_server.app.models.v1 import Task
-from fractal_server.app.schemas.v1 import TaskCollectStatusV1
-from fractal_server.app.schemas.v1 import TaskCreateV1
-from fractal_server.app.schemas.v1 import TaskReadV1
-from fractal_server.logger import close_logger
-from fractal_server.logger import get_logger
-from fractal_server.logger import set_logger
-from fractal_server.utils import execute_command_async
-
-
-async def _pip_install(
-    venv_path: Path,
-    task_pkg: _TaskCollectPip,
-    logger_name: str,
-) -> Path:
-    """
-    Install package in venv
-
-    Args:
-        venv_path:
-        task_pkg:
-        logger_name:
-
-    Returns:
-        The location of the package.
-    """
-
-    logger = get_logger(logger_name)
-
-    pip = venv_path / "venv/bin/pip"
-
-    extras = f"[{task_pkg.package_extras}]" if task_pkg.package_extras else ""
-
-    if task_pkg.is_local_package:
-        pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
-    else:
-        version_string = (
-            f"=={task_pkg.package_version}" if task_pkg.package_version else ""
-        )
-        pip_install_str = f"{task_pkg.package}{extras}{version_string}"
-
-    cmd_install = f"{pip} install {pip_install_str}"
-    cmd_inspect = f"{pip} show {task_pkg.package}"
-
-    await execute_command_async(
-        cwd=venv_path,
-        command=f"{pip} install --upgrade pip",
-        logger_name=logger_name,
-    )
-    await execute_command_async(
-        cwd=venv_path, command=cmd_install, logger_name=logger_name
-    )
-    if task_pkg.pinned_package_versions:
-        for (
-            pinned_pkg_name,
-            pinned_pkg_version,
-        ) in task_pkg.pinned_package_versions.items():
-
-            logger.debug(
-                "Specific version required: "
-                f"{pinned_pkg_name}=={pinned_pkg_version}"
-            )
-            logger.debug(
-                "Preliminary check: verify that "
-                f"{pinned_pkg_version} is already installed"
-            )
-            stdout_inspect = await execute_command_async(
-                cwd=venv_path,
-                command=f"{pip} show {pinned_pkg_name}",
-                logger_name=logger_name,
-            )
-            current_version = next(
-                line.split()[-1]
-                for line in stdout_inspect.split("\n")
-                if line.startswith("Version:")
-            )
-            if current_version != pinned_pkg_version:
-                logger.debug(
-                    f"Currently installed version of {pinned_pkg_name} "
-                    f"({current_version}) differs from pinned version "
-                    f"({pinned_pkg_version}); "
-                    f"install version {pinned_pkg_version}."
-                )
-                await execute_command_async(
-                    cwd=venv_path,
-                    command=(
-                        f"{pip} install "
-                        f"{pinned_pkg_name}=={pinned_pkg_version}"
-                    ),
-                    logger_name=logger_name,
-                )
-            else:
-                logger.debug(
-                    f"Currently installed version of {pinned_pkg_name} "
-                    f"({current_version}) already matches the pinned version."
-                )
-
-    # Extract package installation path from `pip show`
-    stdout_inspect = await execute_command_async(
-        cwd=venv_path, command=cmd_inspect, logger_name=logger_name
-    )
-
-    location = Path(
-        next(
-            line.split()[-1]
-            for line in stdout_inspect.split("\n")
-            if line.startswith("Location:")
-        )
-    )
-
-    # NOTE
-    # https://packaging.python.org/en/latest/specifications/recording-installed-packages/
-    # This directory is named as {name}-{version}.dist-info, with name and
-    # version fields corresponding to Core metadata specifications. Both
-    # fields must be normalized (see the name normalization specification and
-    # the version normalization specification), and replace dash (-)
-    # characters with underscore (_) characters, so the .dist-info directory
-    # always has exactly one dash (-) character in its stem, separating the
-    # name and version fields.
-    package_root = location / (task_pkg.package.replace("-", "_"))
-    logger.debug(f"[_pip install] {location=}")
-    logger.debug(f"[_pip install] {task_pkg.package=}")
-    logger.debug(f"[_pip install] {package_root=}")
-    if not package_root.exists():
-        raise RuntimeError(
-            "Could not determine package installation location."
-        )
-    return package_root
-
-
-async def _create_venv_install_package(
-    *,
-    task_pkg: _TaskCollectPip,
-    path: Path,
-    logger_name: str,
-) -> tuple[Path, Path]:
-    """Create venv and install package
-
-    Args:
-        path: the directory in which to create the environment
-        task_pkg: object containing the different metadata required to install
-            the package
-
-    Returns:
-        python_bin: path to venv's python interpreter
-        package_root: the location of the package manifest
-    """
-
-    # Normalize package name
-    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
-    task_pkg.package = normalize_package_name(task_pkg.package)
-
-    python_bin = await _init_venv_v1(
-        path=path,
-        python_version=task_pkg.python_version,
-        logger_name=logger_name,
-    )
-    package_root = await _pip_install(
-        venv_path=path, task_pkg=task_pkg, logger_name=logger_name
-    )
-    return python_bin, package_root
-
-
-async def create_package_environment_pip(
-    *,
-    task_pkg: _TaskCollectPip,
-    venv_path: Path,
-    logger_name: str,
-) -> list[TaskCreateV1]:
-    """
-    Create environment, install package, and prepare task list
-    """
-
-    logger = get_logger(logger_name)
-
-    # Normalize package name
-    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
-    task_pkg.package = normalize_package_name(task_pkg.package)
-
-    # Only proceed if package, version and manifest attributes are set
-    task_pkg.check()
-
-    try:
-
-        logger.debug("Creating venv and installing package")
-        python_bin, package_root = await _create_venv_install_package(
-            path=venv_path,
-            task_pkg=task_pkg,
-            logger_name=logger_name,
-        )
-        logger.debug("Venv creation and package installation ended correctly.")
-
-        # Prepare task_list with appropriate metadata
-        logger.debug("Creating task list from manifest")
-        task_list = []
-        for t in task_pkg.package_manifest.task_list:
-            # Fill in attributes for TaskCreate
-            task_executable = package_root / t.executable
-            cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
-            task_name_slug = slugify_task_name_for_source_v1(t.name)
-            task_source = f"{task_pkg.package_source}:{task_name_slug}"
-            if not task_executable.exists():
-                raise FileNotFoundError(
-                    f"Cannot find executable `{task_executable}` "
-                    f"for task `{t.name}`"
-                )
-            manifest = task_pkg.package_manifest
-            if manifest.has_args_schemas:
-                additional_attrs = dict(
-                    args_schema_version=manifest.args_schema_version
-                )
-            else:
-                additional_attrs = {}
-            this_task = TaskCreateV1(
-                **t.dict(),
-                command=cmd,
-                version=task_pkg.package_version,
-                **additional_attrs,
-                source=task_source,
-            )
-            task_list.append(this_task)
-        logger.debug("Task list created correctly")
-    except Exception as e:
-        logger.error("Task manifest loading failed")
-        raise e
-    return task_list
-
-
-async def _insert_tasks(
-    task_list: list[TaskCreateV1],
-    db: DBSyncSession,
-) -> list[Task]:
-    """
-    Insert tasks into database
-    """
-    task_db_list = [Task(**t.dict()) for t in task_list]
-    db.add_all(task_db_list)
-    db.commit()
-    for t in task_db_list:
-        db.refresh(t)
-    db.close()
-    return task_db_list
-
-
-async def background_collect_pip(
-    state_id: int,
-    venv_path: Path,
-    task_pkg: _TaskCollectPip,
-) -> None:
-    """
-    Install package and collect tasks
-
-    Install a python package and collect the tasks it provides according to
-    the manifest.
-
-    In case of error, copy the log into the state and delete the package
-    directory.
-    """
-    logger_name = task_pkg.package.replace("/", "_")
-    logger = set_logger(
-        logger_name=logger_name,
-        log_file_path=get_log_path(venv_path),
-    )
-    logger.debug("Start background task collection")
-    for key, value in task_pkg.dict(exclude={"package_manifest"}).items():
-        logger.debug(f"{key}: {value}")
-
-    with next(get_sync_db()) as db:
-        state: State = db.get(State, state_id)
-        data = TaskCollectStatusV1(**state.data)
-        data.info = None
-
-        try:
-            # install
-            logger.debug("Task-collection status: installing")
-            data.status = "installing"
-
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            task_list = await create_package_environment_pip(
-                venv_path=venv_path,
-                task_pkg=task_pkg,
-                logger_name=logger_name,
-            )
-
-            # collect
-            logger.debug("Task-collection status: collecting")
-            data.status = "collecting"
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            tasks = await _insert_tasks(task_list=task_list, db=db)
-
-            # finalise
-            logger.debug("Task-collection status: finalising")
-            collection_path = get_collection_path(venv_path)
-            data.task_list = [
-                TaskReadV1(**task.model_dump()) for task in tasks
-            ]
-            with collection_path.open("w") as f:
-                json.dump(data.sanitised_dict(), f, indent=2)
-
-            # Update DB
-            data.status = "OK"
-            data.log = get_collection_log_v1(venv_path)
-            state.data = data.sanitised_dict()
-            db.add(state)
-            db.merge(state)
-            db.commit()
-
-            # Write last logs to file
-            logger.debug("Task-collection status: OK")
-            logger.info("Background task collection completed successfully")
-            close_logger(logger)
-            db.close()
-
-        except Exception as e:
-            # Write last logs to file
-            logger.debug("Task-collection status: fail")
-            logger.info(f"Background collection failed. Original error: {e}")
-            close_logger(logger)
-
-            # Update db
-            data.status = "fail"
-            data.info = f"Original error: {e}"
-            data.log = get_collection_log_v1(venv_path)
-            state.data = data.sanitised_dict()
-            db.merge(state)
-            db.commit()
-            db.close()
-
-            # Delete corrupted package dir
-            shell_rmtree(venv_path)