fractal-server 2.7.0a0__py3-none-any.whl → 2.7.0a2__py3-none-any.whl

fractal_server/data_migrations/2_7_0.py

@@ -0,0 +1,274 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import TaskGroupV2
+from fractal_server.app.models import TaskV2
+from fractal_server.app.models import UserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
+from fractal_server.app.security import FRACTAL_DEFAULT_GROUP_NAME
+from fractal_server.data_migrations.tools import _check_current_version
+from fractal_server.utils import get_timestamp
+
+logger = logging.getLogger("fix_db")
+
+
+def get_unique_value(list_of_objects: list[dict[str, Any]], key: str):
+    """
+    Loop over `list_of_objects` and extract (unique) value for `key`.
+    """
+    unique_values = set()
+    for this_obj in list_of_objects:
+        this_value = this_obj.get(key, None)
+        unique_values.add(this_value)
+    if len(unique_values) != 1:
+        raise RuntimeError(
+            f"There must be a single taskgroup `{key}`, "
+            f"but {unique_values=}"
+        )
+    return unique_values.pop()
+
+
+def get_users_mapping(db) -> dict[str, int]:
+    logger.warning("START _check_users")
+    print()
+
+    stm_users = select(UserOAuth).order_by(UserOAuth.id)
+    users = db.execute(stm_users).scalars().unique().all()
+    name_to_user_id = {}
+    for user in users:
+        logger.warning(f"START handling user {user.id}: '{user.email}'")
+        # Compute "name" attribute
+        user_settings = db.get(UserSettings, user.user_settings_id)
+        name = user.username or user_settings.slurm_user
+        logger.warning(f"{name=}")
+        # Fail for missing values
+        if name is None:
+            raise ValueError(
+                f"User with {user.id=} and {user.email=} has no "
+                "`username` or `slurm_user` set."
+                "Please fix this issue manually."
+            )
+        # Fail for non-unique values
+        existing_user = name_to_user_id.get(name, None)
+        if existing_user is not None:
+            raise ValueError(
+                f"User with {user.id=} and {user.email=} has same "
+                f"`(username or slurm_user)={name}` as another user. "
+                "Please fix this issue manually."
+            )
+        # Update dictionary
+        name_to_user_id[name] = user.id
+        logger.warning(f"END handling user {user.id}: '{user.email}'")
+        print()
+    logger.warning("END _check_users")
+    print()
+    return name_to_user_id
+
+
+def get_default_user_group_id(db):
+    stm = select(UserGroup.id).where(
+        UserGroup.name == FRACTAL_DEFAULT_GROUP_NAME
+    )
+    res = db.execute(stm)
+    default_group_id = res.scalars().one_or_none()
+    if default_group_id is None:
+        raise RuntimeError("Default user group is missing.")
+    else:
+        return default_group_id
+
+
+def get_default_user_id(db):
+
+    DEFAULT_USER_EMAIL = os.getenv("FRACTAL_V27_DEFAULT_USER_EMAIL")
+    if DEFAULT_USER_EMAIL is None:
+        raise ValueError(
+            "FRACTAL_V27_DEFAULT_USER_EMAIL env variable is not set. "
+            "Please set it to be the email of the user who will own "
+            "all previously-global tasks."
+        )
+
+    stm = select(UserOAuth.id).where(UserOAuth.email == DEFAULT_USER_EMAIL)
+    res = db.execute(stm)
+    default_user_id = res.scalars().one_or_none()
+    if default_user_id is None:
+        raise RuntimeError(
+            f"Default user with email {DEFAULT_USER_EMAIL} is missing."
+        )
+    else:
+        return default_user_id
+
+
+def prepare_task_groups(
+    *,
+    user_mapping: dict[str, int],
+    default_user_group_id: int,
+    default_user_id: int,
+    dry_run: bool,
+    db: Session,
+):
+    stm_tasks = select(TaskV2).order_by(TaskV2.id)
+    res = db.execute(stm_tasks).scalars().all()
+    task_groups = {}
+    for task in res:
+        if (
+            task.source.startswith(("pip_remote", "pip_local"))
+            and task.source.count(":") == 5
+        ):
+            source_fields = task.source.split(":")
+            (
+                collection_mode,
+                pkg_name,
+                version,
+                extras,
+                python_version,
+                name,
+            ) = source_fields
+            task_group_key = ":".join(
+                [pkg_name, version, extras, python_version]
+            )
+            if collection_mode == "pip_remote":
+                origin = "pypi"
+            elif collection_mode == "pip_local":
+                origin = "wheel-file"
+            else:
+                raise RuntimeError(
+                    f"Invalid {collection_mode=} for {task.source=}."
+                )
+            new_obj = dict(
+                task=task,
+                user_id=default_user_id,
+                origin=origin,
+                pkg_name=pkg_name,
+                version=version,
+                pip_extras=extras,
+                python_version=python_version,
+            )
+
+            if task_group_key in task_groups:
+                task_groups[task_group_key].append(new_obj)
+            else:
+                task_groups[task_group_key] = [new_obj]
+        else:
+            owner = task.owner
+            if owner is None:
+                raise RuntimeError(
+                    "Error: `owner` is `None` for "
+                    f"{task.id=}, {task.source=}, {task.owner=}."
+                )
+            user_id = user_mapping.get(owner, None)
+            if user_id is None:
+                raise RuntimeError(
+                    "Error: `user_id` is `None` for "
+                    f"{task.id=}, {task.source=}, {task.owner=}"
+                )
+            task_group_key = "-".join(
+                [
+                    "NOT_PIP",
+                    str(task.id),
+                    str(task.version),
+                    task.source,
+                    str(task.owner),
+                ]
+            )
+            if task_group_key in task_groups:
+                raise RuntimeError(
+                    f"ERROR: Duplicated {task_group_key=} for "
+                    f"{task.id=}, {task.source=}, {task.owner=}"
+                )
+            else:
+                task_groups[task_group_key] = [
+                    dict(
+                        task=task,
+                        user_id=user_id,
+                        origin="other",
+                        pkg_name=task.source,
+                        version=task.version,
+                    )
+                ]
+
+    for task_group_key, task_group_objects in task_groups.items():
+        print("-" * 80)
+        print(f"Start handling task group with key '{task_group_key}")
+        task_group_task_list = [item["task"] for item in task_group_objects]
+        print("List of tasks to be included")
+        for task in task_group_task_list:
+            print(f"  {task.id=}, {task.source=}")
+
+        task_group_attributes = dict(
+            pkg_name=get_unique_value(task_group_objects, "pkg_name"),
+            version=get_unique_value(task_group_objects, "version"),
+            origin=get_unique_value(task_group_objects, "origin"),
+            user_id=get_unique_value(task_group_objects, "user_id"),
+            user_group_id=default_user_group_id,
+            python_version=get_unique_value(
+                task_group_objects, "python_version"
+            ),
+            pip_extras=get_unique_value(task_group_objects, "pip_extras"),
+            task_list=task_group_task_list,
+            active=True,
+            timestamp_created=get_timestamp(),
+        )
+
+        if not task_group_key.startswith("NOT_PIP"):
+            cmd = next(
+                getattr(task_group_task_list[0], attr_name)
+                for attr_name in ["command_non_parallel", "command_parallel"]
+                if getattr(task_group_task_list[0], attr_name) is not None
+            )
+            python_bin = cmd.split()[0]
+            venv_path = Path(python_bin).parents[1].as_posix()
+            path = Path(python_bin).parents[2].as_posix()
+            task_group_attributes["venv_path"] = venv_path
+            task_group_attributes["path"] = path
+
+        print()
+        print("List of task-group attributes")
+        for key, value in task_group_attributes.items():
+            if key != "task_list":
+                print(f"  {key}: {value}")
+
+        print()
+
+        if dry_run:
+            print(
+                "End dry-run of handling task group with key "
+                f"'{task_group_key}"
+            )
+            print("-" * 80)
+            continue
+
+        task_group = TaskGroupV2(**task_group_attributes)
+        db.add(task_group)
+        db.commit()
+        db.refresh(task_group)
+        logger.warning(f"Created task group {task_group.id=}")
+        print()
+
+    return
+
+
+def fix_db(dry_run: bool = False):
+    logger.warning("START execution of fix_db function")
+    _check_current_version("2.7.0")
+
+    with next(get_sync_db()) as db:
+        user_mapping = get_users_mapping(db)
+        default_user_id = get_default_user_id(db)
+        default_user_group_id = get_default_user_group_id(db)
+
+        prepare_task_groups(
+            user_mapping=user_mapping,
+            default_user_id=default_user_id,
+            default_user_group_id=default_user_group_id,
+            db=db,
+            dry_run=dry_run,
+        )
+
+    logger.warning("END of execution of fix_db function")

fractal_server/migrations/versions/742b74e1cc6e_revamp_taskv2_and_taskgroupv2.py

@@ -0,0 +1,101 @@
+"""Revamp TaskV2 and TaskGroupV2
+
+Revision ID: 742b74e1cc6e
+Revises: df7cc3501bf7
+Create Date: 2024-10-07 16:56:37.399878
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "742b74e1cc6e"
+down_revision = "df7cc3501bf7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "origin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "pkg_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "python_version",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "venv_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "pip_extras", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "category", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "modality", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "authors", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column("tags", sa.JSON(), server_default="[]", nullable=False)
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_column("tags")
+        batch_op.drop_column("authors")
+        batch_op.drop_column("modality")
+        batch_op.drop_column("category")
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_column("pip_extras")
+        batch_op.drop_column("venv_path")
+        batch_op.drop_column("path")
+        batch_op.drop_column("python_version")
+        batch_op.drop_column("version")
+        batch_op.drop_column("pkg_name")
+        batch_op.drop_column("origin")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/df7cc3501bf7_linkusergroup_timestamp_created.py

@@ -0,0 +1,42 @@
+"""LinkUserGroup.timestamp_created
+
+Revision ID: df7cc3501bf7
+Revises: 7cf1baae8fb4
+Create Date: 2024-10-03 13:55:53.272269
+
+"""
+from datetime import datetime
+from datetime import timezone
+
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "df7cc3501bf7"
+down_revision = "7cf1baae8fb4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "timestamp_created",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=str(datetime(2000, 1, 1, tzinfo=timezone.utc)),
+            )
+        )
+
+    with op.batch_alter_table("project", schema=None) as batch_op:
+        batch_op.alter_column("timestamp_created", server_default=None)
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_column("timestamp_created")
+
+    # ### end Alembic commands ###

fractal_server/tasks/v2/background_operations.py

@@ -172,6 +172,7 @@ def _prepare_tasks_metadata(
                 }
             ),
             **task_attributes,
+            authors=package_manifest.authors,
         )
         task_list.append(task_obj)
     return task_list
@@ -274,9 +275,19 @@ async def background_collect_pip(
             )
             _check_task_files_exist(task_list=task_list)
 
+            # Prepare some task-group attributes
+            task_group_attrs = dict(
+                pkg_name=task_pkg.package_name,
+                version=task_pkg.package_version,
+            )
+            if task_pkg.is_local_package:
+                task_group_attrs["origin"] = "wheel-file"
+            else:
+                task_group_attrs["origin"] = "pypi"
+
             task_group = create_db_task_group_and_tasks(
                 task_list=task_list,
-                task_group_obj=TaskGroupCreateV2(),
+                task_group_obj=TaskGroupCreateV2(**task_group_attrs),
                 user_id=user_id,
                 user_group_id=user_group_id,
                 db=db,

fractal_server/tasks/v2/background_operations_ssh.py

@@ -316,9 +316,19 @@ def background_collect_pip_ssh(
                     python_bin=Path(python_bin),
                 )
 
+                # Prepare some task-group attributes
+                task_group_attrs = dict(
+                    pkg_name=task_pkg.package_name,
+                    version=task_pkg.package_version,
+                )
+                if task_pkg.is_local_package:
+                    task_group_attrs["origin"] = "wheel-file"
+                else:
+                    task_group_attrs["origin"] = "pypi"
+
                 create_db_task_group_and_tasks(
                     task_list=task_list,
-                    task_group_obj=TaskGroupCreateV2(),
+                    task_group_obj=TaskGroupCreateV2(**task_group_attrs),
                     user_id=user_id,
                     user_group_id=user_group_id,
                     db=db,

fractal_server/tasks/v2/endpoint_operations.py

@@ -5,17 +5,24 @@ from typing import Optional
 from typing import Union
 from zipfile import ZipFile
 
+from fastapi import HTTPException
+from fastapi import status
+from httpx import AsyncClient
+from httpx import TimeoutException
+
 from ._TaskCollectPip import _TaskCollectPip
 from .utils import _parse_wheel_filename
 from .utils import get_python_interpreter_v2
 from fractal_server.app.schemas.v2 import ManifestV2
 from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
+from fractal_server.logger import set_logger
 from fractal_server.syringe import Inject
 from fractal_server.utils import execute_command
 
 
 FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
+logger = set_logger(__name__)
 
 
 async def download_package(
@@ -134,3 +141,38 @@ def create_package_dir_pip(
     if create:
         venv_path.mkdir(exist_ok=False, parents=True)
     return venv_path
+
+
+async def get_package_version_from_pypi(name: str) -> str:
+    """
+    Make a GET call to PyPI JSON API and get latest package version.
+
+    Ref https://warehouse.pypa.io/api-reference/json.html.
+
+    Arguments:
+        name: Package name.
+    """
+
+    url = f"https://pypi.org/pypi/{name}/json"
+    hint = f"Hint: specify the required version for '{name}'."
+    try:
+        async with AsyncClient(timeout=5.0) as client:
+            res = await client.get(url)
+            if res.status_code != 200:
+                raise HTTPException(
+                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                    detail=(
+                        f"Could not get {url} (status_code {res.status_code})."
+                        f"\n{hint}"
+                    ),
+                )
+            version = res.json()["info"]["version"]
+            return version
+    except (KeyError, TimeoutException) as e:
+        logger.warning(
+            f"An error occurred while getting {url}. Original error: {str(e)}."
+        )
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=(f"An error occurred while getting {url}.\n{hint}"),
+        )

{fractal_server-2.7.0a0.dist-info → fractal_server-2.7.0a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.7.0a0
+Version: 2.7.0a2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause