fractal-server 2.7.0a3__py3-none-any.whl → 2.7.0a5__py3-none-any.whl

fractal_server/images/models.py

@@ -3,6 +3,7 @@ from typing import Optional
 from typing import Union
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
@@ -109,13 +110,10 @@ class SingleImageUpdate(BaseModel):
     _types = validator("types", allow_reuse=True)(valdictkeys("types"))
 
 
-class Filters(BaseModel):
+class Filters(BaseModel, extra=Extra.forbid):
     attributes: dict[str, Any] = Field(default_factory=dict)
     types: dict[str, bool] = Field(default_factory=dict)
 
-    class Config:
-        extra = "forbid"
-
     # Validators
     _attributes = validator("attributes", allow_reuse=True)(
         valdictkeys("attributes")

fractal_server/main.py

@@ -20,7 +20,7 @@ from contextlib import asynccontextmanager
 
 from fastapi import FastAPI
 
-from .app.routes.aux._runner import _backend_supports_shutdown  # FIXME: change
+from .app.routes.aux._runner import _backend_supports_shutdown
 from .app.runner.shutdown import cleanup_after_shutdown
 from .config import get_settings
 from .logger import config_uvicorn_loggers

fractal_server/migrations/versions/034a469ec2eb_task_groups.py

@@ -0,0 +1,184 @@
+"""task groups
+
+Revision ID: 034a469ec2eb
+Revises: da2cb2ac4255
+Create Date: 2024-10-10 16:14:13.976231
+
+"""
+from datetime import datetime
+from datetime import timezone
+
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "034a469ec2eb"
+down_revision = "da2cb2ac4255"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "taskgroupv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("user_group_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "origin", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "pkg_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "python_version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column("path", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "venv_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "wheel_path", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "pip_extras", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "pinned_package_versions",
+            sa.JSON(),
+            server_default="{}",
+            nullable=True,
+        ),
+        sa.Column("active", sa.Boolean(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_group_id"],
+            ["usergroup.id"],
+            name=op.f("fk_taskgroupv2_user_group_id_usergroup"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user_oauth.id"],
+            name=op.f("fk_taskgroupv2_user_id_user_oauth"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_taskgroupv2")),
+    )
+    with op.batch_alter_table("collectionstatev2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("taskgroupv2_id", sa.Integer(), nullable=True)
+        )
+        batch_op.create_foreign_key(
+            batch_op.f("fk_collectionstatev2_taskgroupv2_id_taskgroupv2"),
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+        )
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "timestamp_created",
+                sa.DateTime(timezone=True),
+                nullable=False,
+                server_default=str(datetime(2000, 1, 1, tzinfo=timezone.utc)),
+            )
+        )
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("taskgroupv2_id", sa.Integer(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column(
+                "category", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "modality", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "authors", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+            )
+        )
+        batch_op.add_column(
+            sa.Column("tags", sa.JSON(), server_default="[]", nullable=False)
+        )
+        batch_op.alter_column(
+            "source", existing_type=sa.VARCHAR(), nullable=True
+        )
+
+    try:
+        with op.batch_alter_table("taskv2", schema=None) as batch_op:
+            batch_op.drop_constraint("uq_taskv2_source", type_="unique")
+    except BaseException as e:
+        if op.get_bind().dialect.name != "sqlite":
+            raise e
+        import sqlite3
+        import logging
+
+        logger = logging.getLogger("alembic.runtime.migration")
+        logger.warning(
+            f"Using sqlite, with {sqlite3.version=} and "
+            f"{sqlite3.sqlite_version=}"
+        )
+
+        logger.warning(
+            "Could not drop 'uq_taskv2_source' constraint; this is expected "
+            "when the database was created before the naming convention "
+            "was added."
+        )
+        logger.warning(
+            "As a workaround, we recreate the constraint before dropping it."
+        )
+        with op.batch_alter_table("taskv2", schema=None) as batch_op:
+            batch_op.create_unique_constraint("uq_taskv2_source", ["source"])
+            batch_op.drop_constraint("uq_taskv2_source", type_="unique")
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.create_foreign_key(
+            batch_op.f("fk_taskv2_taskgroupv2_id_taskgroupv2"),
+            "taskgroupv2",
+            ["taskgroupv2_id"],
+            ["id"],
+        )
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_taskv2_taskgroupv2_id_taskgroupv2"),
+            type_="foreignkey",
+        )
+        batch_op.create_unique_constraint("uq_taskv2_source", ["source"])
+        batch_op.alter_column(
+            "source", existing_type=sa.VARCHAR(), nullable=False
+        )
+        batch_op.drop_column("tags")
+        batch_op.drop_column("authors")
+        batch_op.drop_column("modality")
+        batch_op.drop_column("category")
+        batch_op.drop_column("taskgroupv2_id")
+
+    with op.batch_alter_table("linkusergroup", schema=None) as batch_op:
+        batch_op.drop_column("timestamp_created")
+
+    with op.batch_alter_table("collectionstatev2", schema=None) as batch_op:
+        batch_op.drop_constraint(
+            batch_op.f("fk_collectionstatev2_taskgroupv2_id_taskgroupv2"),
+            type_="foreignkey",
+        )
+        batch_op.drop_column("taskgroupv2_id")
+
+    op.drop_table("taskgroupv2")
+    # ### end Alembic commands ###

fractal_server/string_tools.py

@@ -33,14 +33,18 @@ def sanitize_string(value: str) -> str:
     return new_value
 
 
-def slugify_task_name_for_source(task_name: str) -> str:
+def slugify_task_name_for_source_v1(task_name: str) -> str:
     """
     NOTE: this function is used upon creation of tasks' sources, therefore
     for the moment we cannot replace it with its more comprehensive version
     from `fractal_server.string_tools.sanitize_string`, nor we can remove it.
-    As 2.3.1, we are renaming it to `slugify_task_name_for_source`, to make
+
+    As of 2.3.1, we are renaming it to `slugify_task_name_for_source`, to make
     it clear that it should not be used for other purposes.
 
+    As of 2.7.0, we are renaming it to `slugify_task_name_for_source_v1`, to
+    make it clear that it is not used for v2.
+
     Args:
         task_name:
 

fractal_server/tasks/utils.py

@@ -9,9 +9,11 @@ COLLECTION_LOG_FILENAME = "collection.log"
 COLLECTION_FREEZE_FILENAME = "collection_freeze.txt"
 
 
-def get_absolute_venv_path(venv_path: Path) -> Path:
+def get_absolute_venv_path_v1(venv_path: Path) -> Path:
     """
     If a path is not absolute, make it a relative path of FRACTAL_TASKS_DIR.
+
+    As of v2.7.0, we rename this to v1 since it is only to be used in v1.
     """
     if venv_path.is_absolute():
         package_path = venv_path
@@ -33,20 +35,32 @@ def get_freeze_path(base: Path) -> Path:
     return base / COLLECTION_FREEZE_FILENAME
 
 
-def get_collection_log(venv_path: Path) -> str:
-    package_path = get_absolute_venv_path(venv_path)
+def get_collection_log_v1(path: Path) -> str:
+    package_path = get_absolute_venv_path_v1(path)
     log_path = get_log_path(package_path)
     log = log_path.open().read()
     return log
 
 
-def get_collection_freeze(venv_path: Path) -> str:
-    package_path = get_absolute_venv_path(venv_path)
+def get_collection_log_v2(path: Path) -> str:
+    log_path = get_log_path(path)
+    log = log_path.open().read()
+    return log
+
+
+def get_collection_freeze_v1(venv_path: Path) -> str:
+    package_path = get_absolute_venv_path_v1(venv_path)
     freeze_path = get_freeze_path(package_path)
     freeze = freeze_path.open().read()
     return freeze
 
 
+def get_collection_freeze_v2(path: Path) -> str:
+    freeze_path = get_freeze_path(path)
+    freeze = freeze_path.open().read()
+    return freeze
+
+
 def _normalize_package_name(name: str) -> str:
     """
     Implement PyPa specifications for package-name normalization

fractal_server/tasks/v1/_TaskCollectPip.py

@@ -77,7 +77,7 @@ class _TaskCollectPip(TaskCollectPipV1):
         if self.python_version:
             python_version = f"py{self.python_version}"
         else:
-            python_version = ""  # FIXME: can we allow this?
+            python_version = ""
 
         source = ":".join(
             (

fractal_server/tasks/v1/background_operations.py

@@ -6,9 +6,9 @@ import json
 from pathlib import Path
 from shutil import rmtree as shell_rmtree
 
-from ...string_tools import slugify_task_name_for_source
+from ...string_tools import slugify_task_name_for_source_v1
 from ..utils import _normalize_package_name
-from ..utils import get_collection_log
+from ..utils import get_collection_log_v1
 from ..utils import get_collection_path
 from ..utils import get_log_path
 from ._TaskCollectPip import _TaskCollectPip
@@ -215,7 +215,7 @@ async def create_package_environment_pip(
             # Fill in attributes for TaskCreate
             task_executable = package_root / t.executable
             cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
-            task_name_slug = slugify_task_name_for_source(t.name)
+            task_name_slug = slugify_task_name_for_source_v1(t.name)
             task_source = f"{task_pkg.package_source}:{task_name_slug}"
             if not task_executable.exists():
                 raise FileNotFoundError(
@@ -321,7 +321,7 @@ async def background_collect_pip(
 
             # Update DB
             data.status = "OK"
-            data.log = get_collection_log(venv_path)
+            data.log = get_collection_log_v1(venv_path)
             state.data = data.sanitised_dict()
             db.add(state)
             db.merge(state)
@@ -342,7 +342,7 @@ async def background_collect_pip(
             # Update db
             data.status = "fail"
             data.info = f"Original error: {e}"
-            data.log = get_collection_log(venv_path)
+            data.log = get_collection_log_v1(venv_path)
             state.data = data.sanitised_dict()
             db.merge(state)
             db.commit()

fractal_server/tasks/v1/get_collection_data.py

@@ -2,12 +2,12 @@ import json
 from pathlib import Path
 
 from fractal_server.app.schemas.v1 import TaskCollectStatusV1
-from fractal_server.tasks.utils import get_absolute_venv_path
+from fractal_server.tasks.utils import get_absolute_venv_path_v1
 from fractal_server.tasks.utils import get_collection_path
 
 
 def get_collection_data(venv_path: Path) -> TaskCollectStatusV1:
-    package_path = get_absolute_venv_path(venv_path)
+    package_path = get_absolute_venv_path_v1(venv_path)
     collection_path = get_collection_path(package_path)
     with collection_path.open() as f:
         data = json.load(f)

fractal_server/tasks/v2/_venv_pip.py

@@ -2,17 +2,48 @@ from pathlib import Path
 from typing import Optional
 
 from ..utils import COLLECTION_FREEZE_FILENAME
+from fractal_server.app.models.v2 import TaskGroupV2
 from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
-from fractal_server.tasks.v2._TaskCollectPip import _TaskCollectPip
 from fractal_server.tasks.v2.utils import get_python_interpreter_v2
 from fractal_server.utils import execute_command
 
 
-async def _pip_install(
+async def _init_venv_v2(
+    *,
     venv_path: Path,
-    task_pkg: _TaskCollectPip,
+    python_version: Optional[str] = None,
+    logger_name: str,
+) -> Path:
+    """
+    Set a virtual environment at `path/venv`
+
+    Args:
+        path : Path
+            path to the venv actual directory (not its parent).
+        python_version : default=None
+            Python version the virtual environment will be based upon
+
+    Returns:
+        python_bin : Path
+            path to python interpreter
+    """
+    logger = get_logger(logger_name)
+    logger.debug(f"[_init_venv_v2] {venv_path=}")
+    interpreter = get_python_interpreter_v2(python_version=python_version)
+    logger.debug(f"[_init_venv_v2] {interpreter=}")
+    await execute_command(
+        command=f"{interpreter} -m venv {venv_path}",
+        logger_name=logger_name,
+    )
+    python_bin = venv_path / "bin/python"
+    logger.debug(f"[_init_venv_v2] {python_bin=}")
+    return python_bin
+
+
+async def _pip_install(
+    task_group: TaskGroupV2,
     logger_name: str,
 ) -> Path:
     """
@@ -30,48 +61,40 @@ async def _pip_install(
 
     logger = get_logger(logger_name)
 
-    pip = venv_path / "venv/bin/pip"
-
-    extras = f"[{task_pkg.package_extras}]" if task_pkg.package_extras else ""
-
-    if task_pkg.is_local_package:
-        pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
-    else:
-        version_string = (
-            f"=={task_pkg.package_version}" if task_pkg.package_version else ""
-        )
-        pip_install_str = f"{task_pkg.package_name}{extras}{version_string}"
+    python_bin = Path(task_group.venv_path) / "bin/python"
+    pip_install_str = task_group.pip_install_string
+    logger.info(f"{pip_install_str=}")
 
     await execute_command(
-        cwd=venv_path,
+        cwd=Path(task_group.venv_path),
         command=(
-            f"{pip} install --upgrade "
+            f"{python_bin} -m pip install --upgrade "
             f"'pip<={settings.FRACTAL_MAX_PIP_VERSION}'"
         ),
         logger_name=logger_name,
     )
     await execute_command(
-        cwd=venv_path,
-        command=f"{pip} install {pip_install_str}",
+        cwd=Path(task_group.venv_path),
+        command=f"{python_bin} -m pip install {pip_install_str}",
         logger_name=logger_name,
     )
-    if task_pkg.pinned_package_versions:
+
+    if task_group.pinned_package_versions:
         for (
             pinned_pkg_name,
             pinned_pkg_version,
-        ) in task_pkg.pinned_package_versions.items():
-
+        ) in task_group.pinned_package_versions.items():
             logger.debug(
                 "Specific version required: "
                 f"{pinned_pkg_name}=={pinned_pkg_version}"
             )
             logger.debug(
                 "Preliminary check: verify that "
-                f"{pinned_pkg_version} is already installed"
+                f"{pinned_pkg_name} is already installed"
             )
             stdout_show = await execute_command(
-                cwd=venv_path,
-                command=f"{pip} show {pinned_pkg_name}",
+                cwd=Path(task_group.venv_path),
+                command=f"{python_bin} -m pip show {pinned_pkg_name}",
                 logger_name=logger_name,
             )
             current_version = next(
@@ -87,9 +110,9 @@ async def _pip_install(
                     f"install version {pinned_pkg_version}."
                 )
                 await execute_command(
-                    cwd=venv_path,
+                    cwd=Path(task_group.venv_path),
                     command=(
-                        f"{pip} install "
+                        f"{python_bin} -m pip install "
                         f"{pinned_pkg_name}=={pinned_pkg_version}"
                     ),
                     logger_name=logger_name,
@@ -102,8 +125,8 @@ async def _pip_install(
 
     # Extract package installation path from `pip show`
     stdout_show = await execute_command(
-        cwd=venv_path,
-        command=f"{pip} show {task_pkg.package_name}",
+        cwd=Path(task_group.venv_path),
+        command=f"{python_bin} -m pip show {task_group.pkg_name}",
         logger_name=logger_name,
     )
 
@@ -124,58 +147,26 @@ async def _pip_install(
     # characters with underscore (_) characters, so the .dist-info directory
     # always has exactly one dash (-) character in its stem, separating the
     # name and version fields.
-    package_root = location / (task_pkg.package_name.replace("-", "_"))
+    package_root = location / (task_group.pkg_name.replace("-", "_"))
     logger.debug(f"[_pip install] {location=}")
-    logger.debug(f"[_pip install] {task_pkg.package_name=}")
+    logger.debug(f"[_pip install] {task_group.pkg_name=}")
     logger.debug(f"[_pip install] {package_root=}")
 
     # Run `pip freeze --all` and store its output
     stdout_freeze = await execute_command(
-        cwd=venv_path, command=f"{pip} freeze --all", logger_name=logger_name
+        cwd=Path(task_group.venv_path),
+        command=f"{python_bin} -m pip freeze --all",
+        logger_name=logger_name,
     )
-    with (venv_path / COLLECTION_FREEZE_FILENAME).open("w") as f:
+    with (Path(task_group.path) / COLLECTION_FREEZE_FILENAME).open("w") as f:
         f.write(stdout_freeze)
 
     return package_root
 
 
-async def _init_venv_v2(
-    *,
-    path: Path,
-    python_version: Optional[str] = None,
-    logger_name: str,
-) -> Path:
-    """
-    Set a virtual environment at `path/venv`
-
-    Args:
-        path : Path
-            path to directory in which to set up the virtual environment
-        python_version : default=None
-            Python version the virtual environment will be based upon
-
-    Returns:
-        python_bin : Path
-            path to python interpreter
-    """
-    logger = get_logger(logger_name)
-    logger.debug(f"[_init_venv] {path=}")
-    interpreter = get_python_interpreter_v2(python_version=python_version)
-    logger.debug(f"[_init_venv] {interpreter=}")
-    await execute_command(
-        cwd=path,
-        command=f"{interpreter} -m venv venv",
-        logger_name=logger_name,
-    )
-    python_bin = path / "venv/bin/python"
-    logger.debug(f"[_init_venv] {python_bin=}")
-    return python_bin
-
-
 async def _create_venv_install_package_pip(
     *,
-    task_pkg: _TaskCollectPip,
-    path: Path,
+    task_group: TaskGroupV2,
     logger_name: str,
 ) -> tuple[Path, Path]:
     """
@@ -191,11 +182,12 @@ async def _create_venv_install_package_pip(
         package_root: the location of the package manifest
     """
     python_bin = await _init_venv_v2(
-        path=path,
-        python_version=task_pkg.python_version,
+        venv_path=Path(task_group.venv_path),
+        python_version=task_group.python_version,
         logger_name=logger_name,
     )
     package_root = await _pip_install(
-        venv_path=path, task_pkg=task_pkg, logger_name=logger_name
+        task_group=task_group,
+        logger_name=logger_name,
     )
     return python_bin, package_root