fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py

@@ -0,0 +1,68 @@
+"""TMP - split Task and WFTask meta
+
+Revision ID: 876f28db9d4e
+Revises: b9e9eed9d442
+Create Date: 2024-03-27 14:35:50.076725
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "876f28db9d4e"
+down_revision = "b9e9eed9d442"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("meta_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("meta_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("meta")
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("meta_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("meta_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("meta")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "meta",
+                postgresql.JSON(astext_type=sa.Text()),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+        batch_op.drop_column("meta_non_parallel")
+        batch_op.drop_column("meta_parallel")
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "meta",
+                postgresql.JSON(astext_type=sa.Text()),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+        batch_op.drop_column("meta_parallel")
+        batch_op.drop_column("meta_non_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py

@@ -0,0 +1,37 @@
+"""TMP - WorkflowTaskV2.type in DB
+
+Revision ID: 974c802f0dd0
+Revises: e3e639454d4b
+Create Date: 2024-04-02 11:21:20.212530
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "974c802f0dd0"
+down_revision = "e3e639454d4b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "task_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_column("task_type")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py

@@ -0,0 +1,40 @@
+"""TMP - WorkflowTaskV2
+
+Revision ID: 9cd305cd6023
+Revises: a6231ed6273c
+Create Date: 2024-03-26 09:15:00.188036
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision = "9cd305cd6023"
+down_revision = "a6231ed6273c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("args_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("args", sqlite.JSON(), nullable=True))
+        batch_op.drop_column("args_non_parallel")
+        batch_op.drop_column("args_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py

@@ -0,0 +1,42 @@
+"""TMP - Args Schemas in TaskV2
+
+Revision ID: a6231ed6273c
+Revises: 56af171b0159
+Create Date: 2024-03-26 08:49:51.870087
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision = "a6231ed6273c"
+down_revision = "56af171b0159"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_schema_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("args_schema_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args_schema")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_schema", sqlite.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args_schema_parallel")
+        batch_op.drop_column("args_schema_non_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py

@@ -0,0 +1,37 @@
+"""TMP - TaskV2.type
+
+Revision ID: b9e9eed9d442
+Revises: 9cd305cd6023
+Create Date: 2024-03-27 13:10:34.125503
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b9e9eed9d442"
+down_revision = "9cd305cd6023"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_column("type")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py

@@ -0,0 +1,50 @@
+"""TMP - make task.meta non optional
+
+Revision ID: e3e639454d4b
+Revises: 4b35c5cefbe3
+Create Date: 2024-03-29 17:10:57.643561
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e3e639454d4b"
+down_revision = "4b35c5cefbe3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+        )
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+        )
+
+    # ### end Alembic commands ###

fractal_server/tasks/__init__.py

@@ -1,8 +1,3 @@
 """
 `tasks` module
-
-Submodules:
-    * [Endpoint functions](background_operations)
-    * [Background-task functions](endpoint_operations)
-    * [Auxiliary functions](utils)
 """

fractal_server/tasks/endpoint_operations.py

@@ -4,35 +4,26 @@ from typing import Optional
 from typing import Union
 from zipfile import ZipFile
 
-from fractal_server.app.schemas import ManifestV1
-from fractal_server.app.schemas import TaskCollectStatus
+from .utils import _normalize_package_name
+from .utils import get_python_interpreter
+from .v1._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV1
+from .v2._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV2
+from fractal_server.app.schemas.v1 import ManifestV1
+from fractal_server.app.schemas.v2 import ManifestV2
 from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
-from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
-from fractal_server.tasks.utils import _normalize_package_name
-from fractal_server.tasks.utils import get_absolute_venv_path
-from fractal_server.tasks.utils import get_collection_path
-from fractal_server.tasks.utils import get_python_interpreter
 from fractal_server.utils import execute_command
 
 
 FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
 
 
-def get_collection_data(venv_path: Path) -> TaskCollectStatus:
-    package_path = get_absolute_venv_path(venv_path)
-    collection_path = get_collection_path(package_path)
-    with collection_path.open() as f:
-        data = json.load(f)
-    return TaskCollectStatus(**data)
-
-
 async def download_package(
     *,
-    task_pkg: _TaskCollectPip,
+    task_pkg: Union[_TaskCollectPipV1, _TaskCollectPipV2],
     dest: Union[str, Path],
-):
+) -> Path:
     """
     Download package to destination
     """
@@ -52,7 +43,7 @@ async def download_package(
 
 def _load_manifest_from_wheel(
     path: Path, wheel: ZipFile, logger_name: Optional[str] = None
-) -> ManifestV1:
+) -> Union[ManifestV1, ManifestV2]:
     logger = get_logger(logger_name)
     namelist = wheel.namelist()
     try:
@@ -69,6 +60,9 @@ def _load_manifest_from_wheel(
     if manifest_version == "1":
         pkg_manifest = ManifestV1(**manifest_dict)
         return pkg_manifest
+    elif manifest_version == "2":
+        pkg_manifest = ManifestV2(**manifest_dict)
+        return pkg_manifest
     else:
         msg = f"Manifest version {manifest_version=} not supported"
         logger.error(msg)
@@ -146,7 +140,7 @@ def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
 
 def create_package_dir_pip(
     *,
-    task_pkg: _TaskCollectPip,
+    task_pkg: Union[_TaskCollectPipV1, _TaskCollectPipV2],
     create: bool = True,
 ) -> Path:
     """

fractal_server/tasks/utils.py

@@ -5,7 +5,9 @@ from pathlib import Path
 from typing import Optional
 
 from fractal_server.config import get_settings
+from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
+from fractal_server.utils import execute_command
 
 COLLECTION_FILENAME = "collection.json"
 COLLECTION_LOG_FILENAME = "collection.log"
@@ -84,3 +86,36 @@ def _normalize_package_name(name: str) -> str:
         The normalized package name.
     """
     return re.sub(r"[-_.]+", "-", name).lower()
+
+
+async def _init_venv(
+    *,
+    path: Path,
+    python_version: Optional[str] = None,
+    logger_name: str,
+) -> Path:
+    """
+    Set a virtual environment at `path/venv`
+
+    Args:
+        path : Path
+            path to directory in which to set up the virtual environment
+        python_version : default=None
+            Python version the virtual environment will be based upon
+
+    Returns:
+        python_bin : Path
+            path to python interpreter
+    """
+    logger = get_logger(logger_name)
+    logger.debug(f"[_init_venv] {path=}")
+    interpreter = get_python_interpreter(version=python_version)
+    logger.debug(f"[_init_venv] {interpreter=}")
+    await execute_command(
+        cwd=path,
+        command=f"{interpreter} -m venv venv",
+        logger_name=logger_name,
+    )
+    python_bin = path / "venv/bin/python"
+    logger.debug(f"[_init_venv] {python_bin=}")
+    return python_bin

fractal_server/tasks/{_TaskCollectPip.py → v1/_TaskCollectPip.py}

@@ -3,11 +3,11 @@ from typing import Optional
 
 from pydantic import root_validator
 
-from fractal_server.app.schemas import ManifestV1
-from fractal_server.app.schemas import TaskCollectPip
+from fractal_server.app.schemas.v1 import ManifestV1
+from fractal_server.app.schemas.v1 import TaskCollectPipV1
 
 
-class _TaskCollectPip(TaskCollectPip):
+class _TaskCollectPip(TaskCollectPipV1):
     """
     Internal TaskCollectPip schema
 

fractal_server/tasks/{background_operations.py → v1/background_operations.py}

@@ -5,61 +5,27 @@ is used as a background task for the task-collection endpoint.
 import json
 from pathlib import Path
 from shutil import rmtree as shell_rmtree
-from typing import Optional
 
+from ..utils import _init_venv
+from ..utils import _normalize_package_name
+from ..utils import get_collection_log
+from ..utils import get_collection_path
+from ..utils import get_log_path
+from ..utils import slugify_task_name
+from ._TaskCollectPip import _TaskCollectPip
 from fractal_server.app.db import DBSyncSession
 from fractal_server.app.db import get_sync_db
 from fractal_server.app.models import State
 from fractal_server.app.models import Task
-from fractal_server.app.schemas import TaskCollectStatus
-from fractal_server.app.schemas import TaskCreate
-from fractal_server.app.schemas import TaskRead
+from fractal_server.app.schemas.v1 import TaskCollectStatusV1
+from fractal_server.app.schemas.v1 import TaskCreateV1
+from fractal_server.app.schemas.v1 import TaskReadV1
 from fractal_server.logger import close_logger
 from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
-from fractal_server.tasks._TaskCollectPip import _TaskCollectPip
-from fractal_server.tasks.utils import _normalize_package_name
-from fractal_server.tasks.utils import get_collection_log
-from fractal_server.tasks.utils import get_collection_path
-from fractal_server.tasks.utils import get_log_path
-from fractal_server.tasks.utils import get_python_interpreter
-from fractal_server.tasks.utils import slugify_task_name
 from fractal_server.utils import execute_command
 
 
-async def _init_venv(
-    *,
-    path: Path,
-    python_version: Optional[str] = None,
-    logger_name: str,
-) -> Path:
-    """
-    Set a virtual environment at `path/venv`
-
-    Args:
-        path : Path
-            path to directory in which to set up the virtual environment
-        python_version : default=None
-            Python version the virtual environment will be based upon
-
-    Returns:
-        python_bin : Path
-            path to python interpreter
-    """
-    logger = get_logger(logger_name)
-    logger.debug(f"[_init_venv] {path=}")
-    interpreter = get_python_interpreter(version=python_version)
-    logger.debug(f"[_init_venv] {interpreter=}")
-    await execute_command(
-        cwd=path,
-        command=f"{interpreter} -m venv venv",
-        logger_name=logger_name,
-    )
-    python_bin = path / "venv/bin/python"
-    logger.debug(f"[_init_venv] {python_bin=}")
-    return python_bin
-
-
 async def _pip_install(
     venv_path: Path,
     task_pkg: _TaskCollectPip,
@@ -218,7 +184,7 @@ async def create_package_environment_pip(
     task_pkg: _TaskCollectPip,
     venv_path: Path,
     logger_name: str,
-) -> list[TaskCreate]:
+) -> list[TaskCreateV1]:
     """
     Create environment, install package, and prepare task list
     """
@@ -263,7 +229,7 @@ async def create_package_environment_pip(
                 )
             else:
                 additional_attrs = {}
-            this_task = TaskCreate(
+            this_task = TaskCreateV1(
                 **t.dict(),
                 command=cmd,
                 version=task_pkg.package_version,
@@ -279,7 +245,7 @@ async def create_package_environment_pip(
 
 
 async def _insert_tasks(
-    task_list: list[TaskCreate],
+    task_list: list[TaskCreateV1],
     db: DBSyncSession,
 ) -> list[Task]:
     """
@@ -319,7 +285,7 @@ async def background_collect_pip(
 
     with next(get_sync_db()) as db:
         state: State = db.get(State, state_id)
-        data = TaskCollectStatus(**state.data)
+        data = TaskCollectStatusV1(**state.data)
         data.info = None
 
         try:
@@ -347,9 +313,11 @@ async def background_collect_pip(
             # finalise
             logger.debug("Task-collection status: finalising")
             collection_path = get_collection_path(venv_path)
-            data.task_list = [TaskRead(**task.model_dump()) for task in tasks]
+            data.task_list = [
+                TaskReadV1(**task.model_dump()) for task in tasks
+            ]
             with collection_path.open("w") as f:
-                json.dump(data.sanitised_dict(), f)
+                json.dump(data.sanitised_dict(), f, indent=2)
 
             # Update DB
             data.status = "OK"

fractal_server/tasks/v1/get_collection_data.py

@@ -0,0 +1,14 @@
+import json
+from pathlib import Path
+
+from fractal_server.app.schemas.v1 import TaskCollectStatusV1
+from fractal_server.tasks.utils import get_absolute_venv_path
+from fractal_server.tasks.utils import get_collection_path
+
+
+def get_collection_data(venv_path: Path) -> TaskCollectStatusV1:
+    package_path = get_absolute_venv_path(venv_path)
+    collection_path = get_collection_path(package_path)
+    with collection_path.open() as f:
+        data = json.load(f)
+    return TaskCollectStatusV1(**data)

fractal_server/tasks/v2/_TaskCollectPip.py

@@ -0,0 +1,103 @@
+from pathlib import Path
+from typing import Optional
+
+from pydantic import root_validator
+
+from fractal_server.app.schemas.v2 import ManifestV2
+from fractal_server.app.schemas.v2 import TaskCollectPipV2
+
+
+class _TaskCollectPip(TaskCollectPipV2):
+    """
+    Internal TaskCollectPip schema
+
+    Differences with its parent class (`TaskCollectPip`):
+
+        1. We check if the package corresponds to a path in the filesystem, and
+           whether it exists (via new validator `check_local_package`, new
+           method `is_local_package` and new attribute `package_path`).
+        2. We include an additional `package_manifest` attribute.
+        3. We expose an additional attribute `package_name`, which is filled
+           during task collection.
+    """
+
+    package_name: Optional[str] = None
+    package_path: Optional[Path] = None
+    package_manifest: Optional[ManifestV2] = None
+
+    @property
+    def is_local_package(self) -> bool:
+        return bool(self.package_path)
+
+    @root_validator(pre=True)
+    def check_local_package(cls, values):
+        """
+        Checks if package corresponds to an existing path on the filesystem
+
+        In this case, the user is providing directly a package file, rather
+        than a remote one from PyPI. We set the `package_path` attribute and
+        get the actual package name and version from the package file name.
+        """
+        if "/" in values["package"]:
+            package_path = Path(values["package"])
+            if not package_path.is_absolute():
+                raise ValueError("Package path must be absolute")
+            if package_path.exists():
+                values["package_path"] = package_path
+                (
+                    values["package"],
+                    values["version"],
+                    *_,
+                ) = package_path.name.split("-")
+            else:
+                raise ValueError(f"Package {package_path} does not exist.")
+        return values
+
+    @property
+    def package_source(self) -> str:
+        """
+        NOTE: As of PR #1188 in `fractal-server`, the attribute
+        `self.package_name` is normalized; this means e.g. that `_` is
+        replaced by `-`. To guarantee backwards compatibility with
+        `Task.source` attributes created before this change, we still replace
+        `-` with `_` upon generation of the `source` attribute, in this
+        method.
+        """
+        if not self.package_name or not self.package_version:
+            raise ValueError(
+                "Cannot construct `package_source` property with "
+                f"{self.package_name=} and {self.package_version=}."
+            )
+        if self.is_local_package:
+            collection_type = "pip_local"
+        else:
+            collection_type = "pip_remote"
+
+        package_extras = self.package_extras or ""
+        if self.python_version:
+            python_version = f"py{self.python_version}"
+        else:
+            python_version = ""  # FIXME: can we allow this?
+
+        source = ":".join(
+            (
+                collection_type,
+                self.package_name.replace("-", "_"),  # see method docstring
+                self.package_version,
+                package_extras,
+                python_version,
+            )
+        )
+        return source
+
+    def check(self):
+        """
+        Verify that the package has all attributes that are needed to continue
+        with task collection
+        """
+        if not self.package_name:
+            raise ValueError("`package_name` attribute is not set")
+        if not self.package_version:
+            raise ValueError("`package_version` attribute is not set")
+        if not self.package_manifest:
+            raise ValueError("`package_manifest` attribute is not set")