fractal-server 2.7.0a1__py3-none-any.whl → 2.7.0a2__py3-none-any.whl

fractal_server/app/routes/api/v2/task.py

@@ -6,6 +6,7 @@ from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Response
 from fastapi import status
+from sqlmodel import func
 from sqlmodel import or_
 from sqlmodel import select
 
@@ -36,6 +37,9 @@ logger = set_logger(__name__)
 async def get_list_task(
     args_schema_parallel: bool = True,
     args_schema_non_parallel: bool = True,
+    category: Optional[str] = None,
+    modality: Optional[str] = None,
+    author: Optional[str] = None,
     user: UserOAuth = Depends(current_active_user),
     db: AsyncSession = Depends(get_async_db),
 ) -> list[TaskReadV2]:
@@ -57,6 +61,13 @@ async def get_list_task(
             )
         )
     )
+    if category is not None:
+        stm = stm.where(func.lower(TaskV2.category) == category.lower())
+    if modality is not None:
+        stm = stm.where(func.lower(TaskV2.modality) == modality.lower())
+    if author is not None:
+        stm = stm.where(TaskV2.authors.icontains(author))
+
     res = await db.execute(stm)
     task_list = res.scalars().all()
     await db.close()
@@ -216,6 +227,8 @@ async def create_task(
         user_group_id=user_group_id,
         active=True,
         task_list=[db_task],
+        origin="other",
+        pkg_name=task.name,
     )
     db.add(db_task_group)
     await db.commit()

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -186,9 +186,15 @@ async def collect_task_custom(
             detail="\n".join(overlapping_tasks_v1_source_and_id),
         )
 
+    # Prepare task-group attributes
+    task_group_attrs = dict(
+        origin="other",
+        pkg_name=task_collect.source,  # FIXME
+    )
+
     task_group = create_db_task_group_and_tasks(
         task_list=task_list,
-        task_group_obj=TaskGroupCreateV2(),
+        task_group_obj=TaskGroupCreateV2(**task_group_attrs),
         user_id=user.id,
         user_group_id=user_group_id,
         db=db_sync,

fractal_server/app/schemas/v2/manifest.py

@@ -7,6 +7,8 @@ from pydantic import HttpUrl
 from pydantic import root_validator
 from pydantic import validator
 
+from .._validators import valstr
+
 
 class TaskManifestV2(BaseModel):
     """
@@ -50,6 +52,10 @@ class TaskManifestV2(BaseModel):
     docs_info: Optional[str] = None
     docs_link: Optional[HttpUrl] = None
 
+    category: Optional[str] = None
+    modality: Optional[str] = None
+    tags: list[str] = Field(default_factory=list)
+
     @root_validator
     def validate_executable_args_meta(cls, values):
 
@@ -128,7 +134,8 @@ class ManifestV2(BaseModel):
     manifest_version: str
     task_list: list[TaskManifestV2]
     has_args_schemas: bool = False
-    args_schema_version: Optional[str]
+    args_schema_version: Optional[str] = None
+    authors: Optional[str] = None
 
     @root_validator()
     def _check_args_schemas_are_present(cls, values):
@@ -157,3 +164,7 @@ class ManifestV2(BaseModel):
         if value != "2":
             raise ValueError(f"Wrong manifest version (given {value})")
         return value
+
+    _authors = validator("authors", allow_reuse=True)(
+        valstr("authors", accept_none=True)
+    )

fractal_server/app/schemas/v2/task.py

@@ -9,8 +9,9 @@ from pydantic import HttpUrl
 from pydantic import root_validator
 from pydantic import validator
 
-from .._validators import valdictkeys
-from .._validators import valstr
+from fractal_server.app.schemas._validators import val_unique_list
+from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.app.schemas._validators import valstr
 from fractal_server.string_tools import validate_cmd
 
 
@@ -18,22 +19,27 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
 
-    command_non_parallel: Optional[str]
-    command_parallel: Optional[str]
+    command_non_parallel: Optional[str] = None
+    command_parallel: Optional[str] = None
     source: str
 
-    meta_non_parallel: Optional[dict[str, Any]]
-    meta_parallel: Optional[dict[str, Any]]
-    version: Optional[str]
-    args_schema_non_parallel: Optional[dict[str, Any]]
-    args_schema_parallel: Optional[dict[str, Any]]
-    args_schema_version: Optional[str]
-    docs_info: Optional[str]
-    docs_link: Optional[HttpUrl]
+    meta_non_parallel: Optional[dict[str, Any]] = None
+    meta_parallel: Optional[dict[str, Any]] = None
+    version: Optional[str] = None
+    args_schema_non_parallel: Optional[dict[str, Any]] = None
+    args_schema_parallel: Optional[dict[str, Any]] = None
+    args_schema_version: Optional[str] = None
+    docs_info: Optional[str] = None
+    docs_link: Optional[HttpUrl] = None
 
     input_types: dict[str, bool] = Field(default={})
     output_types: dict[str, bool] = Field(default={})
 
+    category: Optional[str] = None
+    modality: Optional[str] = None
+    tags: list[str] = Field(default_factory=list)
+    authors: Optional[str] = None
+
     # Validators
     @root_validator
     def validate_commands(cls, values):
@@ -83,6 +89,22 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
         valdictkeys("output_types")
     )
 
+    _category = validator("category", allow_reuse=True)(
+        valstr("category", accept_none=True)
+    )
+    _modality = validator("modality", allow_reuse=True)(
+        valstr("modality", accept_none=True)
+    )
+    _authors = validator("authors", allow_reuse=True)(
+        valstr("authors", accept_none=True)
+    )
+
+    @validator("tags")
+    def validate_list_of_strings(cls, value):
+        for i, tag in enumerate(value):
+            value[i] = valstr(f"tags[{i}]")(tag)
+        return val_unique_list("tags")(value)
+
 
 class TaskReadV2(BaseModel):
 
@@ -90,31 +112,41 @@ class TaskReadV2(BaseModel):
     name: str
     type: Literal["parallel", "non_parallel", "compound"]
     source: str
-    version: Optional[str]
+    version: Optional[str] = None
 
-    command_non_parallel: Optional[str]
-    command_parallel: Optional[str]
+    command_non_parallel: Optional[str] = None
+    command_parallel: Optional[str] = None
     meta_parallel: dict[str, Any]
     meta_non_parallel: dict[str, Any]
     args_schema_non_parallel: Optional[dict[str, Any]] = None
     args_schema_parallel: Optional[dict[str, Any]] = None
-    args_schema_version: Optional[str]
-    docs_info: Optional[str]
-    docs_link: Optional[HttpUrl]
+    args_schema_version: Optional[str] = None
+    docs_info: Optional[str] = None
+    docs_link: Optional[HttpUrl] = None
     input_types: dict[str, bool]
     output_types: dict[str, bool]
 
-    taskgroupv2_id: Optional[int]
+    taskgroupv2_id: Optional[int] = None
+
+    category: Optional[str] = None
+    modality: Optional[str] = None
+    authors: Optional[str] = None
+    tags: list[str]
 
 
 class TaskUpdateV2(BaseModel):
 
-    name: Optional[str]
-    version: Optional[str]
-    command_parallel: Optional[str]
-    command_non_parallel: Optional[str]
-    input_types: Optional[dict[str, bool]]
-    output_types: Optional[dict[str, bool]]
+    name: Optional[str] = None
+    version: Optional[str] = None
+    command_parallel: Optional[str] = None
+    command_non_parallel: Optional[str] = None
+    input_types: Optional[dict[str, bool]] = None
+    output_types: Optional[dict[str, bool]] = None
+
+    category: Optional[str] = None
+    modality: Optional[str] = None
+    authors: Optional[str] = None
+    tags: Optional[list[str]] = None
 
     # Validators
     @validator("input_types", "output_types")
@@ -140,6 +172,22 @@ class TaskUpdateV2(BaseModel):
         valdictkeys("output_types")
     )
 
+    _category = validator("category", allow_reuse=True)(
+        valstr("category", accept_none=True)
+    )
+    _modality = validator("modality", allow_reuse=True)(
+        valstr("modality", accept_none=True)
+    )
+    _authors = validator("authors", allow_reuse=True)(
+        valstr("authors", accept_none=True)
+    )
+
+    @validator("tags")
+    def validate_tags(cls, value):
+        for i, tag in enumerate(value):
+            value[i] = valstr(f"tags[{i}]")(tag)
+        return val_unique_list("tags")(value)
+
 
 class TaskImportV2(BaseModel):
 

fractal_server/app/schemas/v2/task_group.py

@@ -1,23 +1,50 @@
+from datetime import datetime
+from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import validator
 
 from .task import TaskReadV2
 
 
 class TaskGroupCreateV2(BaseModel):
     active: bool = True
+    origin: Literal["pypi", "wheel-file", "other"]
+    pkg_name: str
+    version: Optional[str] = None
+    python_version: Optional[str] = None
+    path: Optional[str] = None
+    venv_path: Optional[str] = None
+    pip_extras: Optional[str] = None
 
 
 class TaskGroupReadV2(BaseModel):
 
     id: int
+    task_list: list[TaskReadV2]
+
     user_id: int
     user_group_id: Optional[int] = None
+
+    origin: Literal["pypi", "wheel-file", "other"]
+    pkg_name: str
+    version: Optional[str] = None
+    python_version: Optional[str] = None
+    path: Optional[str] = None
+    venv_path: Optional[str] = None
+    pip_extras: Optional[str] = None
+
     active: bool
-    task_list: list[TaskReadV2]
+    timestamp_created: datetime
 
 
 class TaskGroupUpdateV2(BaseModel):
     user_group_id: Optional[int] = None
     active: Optional[bool] = None
+
+    @validator("active")
+    def active_cannot_be_None(cls, value):
+        if value is None:
+            raise ValueError("`active` cannot be set to None")
+        return value

fractal_server/data_migrations/2_7_0.py

@@ -0,0 +1,274 @@
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import TaskGroupV2
+from fractal_server.app.models import TaskV2
+from fractal_server.app.models import UserGroup
+from fractal_server.app.models import UserOAuth
+from fractal_server.app.models import UserSettings
+from fractal_server.app.security import FRACTAL_DEFAULT_GROUP_NAME
+from fractal_server.data_migrations.tools import _check_current_version
+from fractal_server.utils import get_timestamp
+
+logger = logging.getLogger("fix_db")
+
+
+def get_unique_value(list_of_objects: list[dict[str, Any]], key: str):
+    """
+    Loop over `list_of_objects` and extract (unique) value for `key`.
+    """
+    unique_values = set()
+    for this_obj in list_of_objects:
+        this_value = this_obj.get(key, None)
+        unique_values.add(this_value)
+    if len(unique_values) != 1:
+        raise RuntimeError(
+            f"There must be a single taskgroup `{key}`, "
+            f"but {unique_values=}"
+        )
+    return unique_values.pop()
+
+
+def get_users_mapping(db) -> dict[str, int]:
+    logger.warning("START _check_users")
+    print()
+
+    stm_users = select(UserOAuth).order_by(UserOAuth.id)
+    users = db.execute(stm_users).scalars().unique().all()
+    name_to_user_id = {}
+    for user in users:
+        logger.warning(f"START handling user {user.id}: '{user.email}'")
+        # Compute "name" attribute
+        user_settings = db.get(UserSettings, user.user_settings_id)
+        name = user.username or user_settings.slurm_user
+        logger.warning(f"{name=}")
+        # Fail for missing values
+        if name is None:
+            raise ValueError(
+                f"User with {user.id=} and {user.email=} has no "
+                "`username` or `slurm_user` set."
+                "Please fix this issue manually."
+            )
+        # Fail for non-unique values
+        existing_user = name_to_user_id.get(name, None)
+        if existing_user is not None:
+            raise ValueError(
+                f"User with {user.id=} and {user.email=} has same "
+                f"`(username or slurm_user)={name}` as another user. "
+                "Please fix this issue manually."
+            )
+        # Update dictionary
+        name_to_user_id[name] = user.id
+        logger.warning(f"END handling user {user.id}: '{user.email}'")
+        print()
+    logger.warning("END _check_users")
+    print()
+    return name_to_user_id
+
+
+def get_default_user_group_id(db):
+    stm = select(UserGroup.id).where(
+        UserGroup.name == FRACTAL_DEFAULT_GROUP_NAME
+    )
+    res = db.execute(stm)
+    default_group_id = res.scalars().one_or_none()
+    if default_group_id is None:
+        raise RuntimeError("Default user group is missing.")
+    else:
+        return default_group_id
+
+
+def get_default_user_id(db):
+
+    DEFAULT_USER_EMAIL = os.getenv("FRACTAL_V27_DEFAULT_USER_EMAIL")
+    if DEFAULT_USER_EMAIL is None:
+        raise ValueError(
+            "FRACTAL_V27_DEFAULT_USER_EMAIL env variable is not set. "
+            "Please set it to be the email of the user who will own "
+            "all previously-global tasks."
+        )
+
+    stm = select(UserOAuth.id).where(UserOAuth.email == DEFAULT_USER_EMAIL)
+    res = db.execute(stm)
+    default_user_id = res.scalars().one_or_none()
+    if default_user_id is None:
+        raise RuntimeError(
+            f"Default user with email {DEFAULT_USER_EMAIL} is missing."
+        )
+    else:
+        return default_user_id
+
+
+def prepare_task_groups(
+    *,
+    user_mapping: dict[str, int],
+    default_user_group_id: int,
+    default_user_id: int,
+    dry_run: bool,
+    db: Session,
+):
+    stm_tasks = select(TaskV2).order_by(TaskV2.id)
+    res = db.execute(stm_tasks).scalars().all()
+    task_groups = {}
+    for task in res:
+        if (
+            task.source.startswith(("pip_remote", "pip_local"))
+            and task.source.count(":") == 5
+        ):
+            source_fields = task.source.split(":")
+            (
+                collection_mode,
+                pkg_name,
+                version,
+                extras,
+                python_version,
+                name,
+            ) = source_fields
+            task_group_key = ":".join(
+                [pkg_name, version, extras, python_version]
+            )
+            if collection_mode == "pip_remote":
+                origin = "pypi"
+            elif collection_mode == "pip_local":
+                origin = "wheel-file"
+            else:
+                raise RuntimeError(
+                    f"Invalid {collection_mode=} for {task.source=}."
+                )
+            new_obj = dict(
+                task=task,
+                user_id=default_user_id,
+                origin=origin,
+                pkg_name=pkg_name,
+                version=version,
+                pip_extras=extras,
+                python_version=python_version,
+            )
+
+            if task_group_key in task_groups:
+                task_groups[task_group_key].append(new_obj)
+            else:
+                task_groups[task_group_key] = [new_obj]
+        else:
+            owner = task.owner
+            if owner is None:
+                raise RuntimeError(
+                    "Error: `owner` is `None` for "
+                    f"{task.id=}, {task.source=}, {task.owner=}."
+                )
+            user_id = user_mapping.get(owner, None)
+            if user_id is None:
+                raise RuntimeError(
+                    "Error: `user_id` is `None` for "
+                    f"{task.id=}, {task.source=}, {task.owner=}"
+                )
+            task_group_key = "-".join(
+                [
+                    "NOT_PIP",
+                    str(task.id),
+                    str(task.version),
+                    task.source,
+                    str(task.owner),
+                ]
+            )
+            if task_group_key in task_groups:
+                raise RuntimeError(
+                    f"ERROR: Duplicated {task_group_key=} for "
+                    f"{task.id=}, {task.source=}, {task.owner=}"
+                )
+            else:
+                task_groups[task_group_key] = [
+                    dict(
+                        task=task,
+                        user_id=user_id,
+                        origin="other",
+                        pkg_name=task.source,
+                        version=task.version,
+                    )
+                ]
+
+    for task_group_key, task_group_objects in task_groups.items():
+        print("-" * 80)
+        print(f"Start handling task group with key '{task_group_key}")
+        task_group_task_list = [item["task"] for item in task_group_objects]
+        print("List of tasks to be included")
+        for task in task_group_task_list:
+            print(f"  {task.id=}, {task.source=}")
+
+        task_group_attributes = dict(
+            pkg_name=get_unique_value(task_group_objects, "pkg_name"),
+            version=get_unique_value(task_group_objects, "version"),
+            origin=get_unique_value(task_group_objects, "origin"),
+            user_id=get_unique_value(task_group_objects, "user_id"),
+            user_group_id=default_user_group_id,
+            python_version=get_unique_value(
+                task_group_objects, "python_version"
+            ),
+            pip_extras=get_unique_value(task_group_objects, "pip_extras"),
+            task_list=task_group_task_list,
+            active=True,
+            timestamp_created=get_timestamp(),
+        )
+
+        if not task_group_key.startswith("NOT_PIP"):
+            cmd = next(
+                getattr(task_group_task_list[0], attr_name)
+                for attr_name in ["command_non_parallel", "command_parallel"]
+                if getattr(task_group_task_list[0], attr_name) is not None
+            )
+            python_bin = cmd.split()[0]
+            venv_path = Path(python_bin).parents[1].as_posix()
+            path = Path(python_bin).parents[2].as_posix()
+            task_group_attributes["venv_path"] = venv_path
+            task_group_attributes["path"] = path
+
+        print()
+        print("List of task-group attributes")
+        for key, value in task_group_attributes.items():
+            if key != "task_list":
+                print(f"  {key}: {value}")
+
+        print()
+
+        if dry_run:
+            print(
+                "End dry-run of handling task group with key "
+                f"'{task_group_key}"
+            )
+            print("-" * 80)
+            continue
+
+        task_group = TaskGroupV2(**task_group_attributes)
+        db.add(task_group)
+        db.commit()
+        db.refresh(task_group)
+        logger.warning(f"Created task group {task_group.id=}")
+        print()
+
+    return
+
+
+def fix_db(dry_run: bool = False):
+    logger.warning("START execution of fix_db function")
+    _check_current_version("2.7.0")
+
+    with next(get_sync_db()) as db:
+        user_mapping = get_users_mapping(db)
+        default_user_id = get_default_user_id(db)
+        default_user_group_id = get_default_user_group_id(db)
+
+        prepare_task_groups(
+            user_mapping=user_mapping,
+            default_user_id=default_user_id,
+            default_user_group_id=default_user_group_id,
+            db=db,
+            dry_run=dry_run,
+        )
+
+    logger.warning("END of execution of fix_db function")

fractal_server/migrations/versions/742b74e1cc6e_revamp_taskv2_and_taskgroupv2.py

@@ -0,0 +1,101 @@
+"""Revamp TaskV2 and TaskGroupV2
+
+Revision ID: 742b74e1cc6e
+Revises: df7cc3501bf7
+Create Date: 2024-10-07 16:56:37.399878
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "742b74e1cc6e"
+down_revision = "df7cc3501bf7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "origin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "pkg_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "python_version",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=True,
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "venv_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "pip_extras", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "category", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "modality", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "authors", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column("tags", sa.JSON(), server_default="[]", nullable=False)
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_column("tags")
+        batch_op.drop_column("authors")
+        batch_op.drop_column("modality")
+        batch_op.drop_column("category")
+
+    with op.batch_alter_table("taskgroupv2", schema=None) as batch_op:
+        batch_op.drop_column("pip_extras")
+        batch_op.drop_column("venv_path")
+        batch_op.drop_column("path")
+        batch_op.drop_column("python_version")
+        batch_op.drop_column("version")
+        batch_op.drop_column("pkg_name")
+        batch_op.drop_column("origin")
+
+    # ### end Alembic commands ###

fractal_server/tasks/v2/background_operations.py

@@ -172,6 +172,7 @@ def _prepare_tasks_metadata(
                 }
             ),
             **task_attributes,
+            authors=package_manifest.authors,
         )
         task_list.append(task_obj)
     return task_list
@@ -274,9 +275,19 @@ async def background_collect_pip(
             )
             _check_task_files_exist(task_list=task_list)
 
+            # Prepare some task-group attributes
+            task_group_attrs = dict(
+                pkg_name=task_pkg.package_name,
+                version=task_pkg.package_version,
+            )
+            if task_pkg.is_local_package:
+                task_group_attrs["origin"] = "wheel-file"
+            else:
+                task_group_attrs["origin"] = "pypi"
+
             task_group = create_db_task_group_and_tasks(
                 task_list=task_list,
-                task_group_obj=TaskGroupCreateV2(),
+                task_group_obj=TaskGroupCreateV2(**task_group_attrs),
                 user_id=user_id,
                 user_group_id=user_group_id,
                 db=db,