fractal-server 1.3.0a2__py3-none-any.whl → 1.3.0a3__py3-none-any.whl

fractal_server/common/tests/test_task.py

@@ -12,6 +12,20 @@ def test_task_update():
     debug(t)
     assert list(t.dict(exclude_none=True).keys()) == ["name"]
     assert list(t.dict(exclude_unset=True).keys()) == ["name"]
+    # Some failures
+    with pytest.raises(ValidationError):
+        TaskUpdate(name="task", version="")
+    with pytest.raises(ValidationError):
+        TaskUpdate(name="task", version=None)
+    # Successful cretion, with mutliple fields set
+    t = TaskUpdate(
+        name="task",
+        version="1.2.3",
+        owner="someone",
+    )
+    debug(t)
+    assert t.name
+    assert t.version
 
 
 def test_task_create():
@@ -22,6 +36,8 @@ def test_task_create():
         command="command",
         input_type="input_type",
         output_type="output_type",
+        version="1.2.3",
+        owner="someone",
     )
     debug(t)
     # Missing arguments

fractal_server/common/tests/test_task_collection.py

@@ -0,0 +1,24 @@
+import pytest
+from devtools import debug
+from pydantic.error_wrappers import ValidationError
+
+from schemas import TaskCollectPip
+
+
+def test_TaskCollectPip():
+    # Successful creation
+    c = TaskCollectPip(package="some-package")
+    debug(c)
+    assert c
+    c = TaskCollectPip(package="/some/package.whl")
+    debug(c)
+    assert c
+    # Failed creation
+    with pytest.raises(ValidationError):
+        c = TaskCollectPip(package="some/package")
+    with pytest.raises(ValidationError):
+        c = TaskCollectPip(package="/some/package.tar.gz")
+    with pytest.raises(ValidationError):
+        c = TaskCollectPip(package="some-package", package_extras="")
+    with pytest.raises(ValidationError):
+        c = TaskCollectPip(package="some-package", package_extras=None)

fractal_server/common/tests/test_user.py

@@ -31,3 +31,15 @@ def test_user_create():
         u = UserCreate(email="a@b.c", password="asd", cache_dir="xxx")
     debug(e.value)
     assert "must be an absolute path" in e.value.errors()[0]["msg"]
+    # With all attributes
+    u = UserCreate(
+        email="a@b.c",
+        password="pwd",
+        slurm_user="slurm_user",
+        username="username",
+        cache_dir="/some/path",
+    )
+    debug(u)
+    assert u.slurm_user
+    assert u.cache_dir
+    assert u.username

fractal_server/main.py

@@ -85,6 +85,7 @@ async def _create_first_user(
     is_superuser: bool = False,
     slurm_user: Optional[str] = None,
     cache_dir: Optional[str] = None,
+    username: Optional[str] = None,
 ) -> None:
     """
     Private method to create the first fractal-server user
@@ -121,6 +122,8 @@ async def _create_first_user(
                         kwargs["slurm_user"] = slurm_user
                     if cache_dir:
                         kwargs["cache_dir"] = cache_dir
+                    if username:
+                        kwargs["username"] = username
                     user = await user_manager.create(UserCreate(**kwargs))
                     logger.info(f"User {user.email} created")
 

fractal_server/migrations/versions/4c308bcaea2b_add_task_args_schema_and_task_args_.py

@@ -0,0 +1,38 @@
+"""Add task.args_schema and task.args_schema_version
+
+Revision ID: 4c308bcaea2b
+Revises: 50a13d6138fd
+Create Date: 2023-05-29 17:09:02.492639
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "4c308bcaea2b"
+down_revision = "50a13d6138fd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("task", sa.Column("args_schema", sa.JSON(), nullable=True))
+    op.add_column(
+        "task",
+        sa.Column(
+            "args_schema_version",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("task", "args_schema_version")
+    op.drop_column("task", "args_schema")
+    # ### end Alembic commands ###

fractal_server/migrations/versions/{e8f4051440be_new_initial_schema.py → 50a13d6138fd_initial_schema.py}

@@ -1,8 +1,8 @@
-"""New initial schema
+"""Initial schema
 
-Revision ID: e8f4051440be
+Revision ID: 50a13d6138fd
 Revises:
-Create Date: 2023-05-08 09:23:14.013706
+Create Date: 2023-05-29 12:14:56.670243
 
 """
 import sqlalchemy as sa
@@ -11,7 +11,7 @@ from alembic import op
 
 
 # revision identifiers, used by Alembic.
-revision = "e8f4051440be"
+revision = "50a13d6138fd"
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -37,11 +37,11 @@ def upgrade() -> None:
         "task",
         sa.Column("default_args", sa.JSON(), nullable=True),
         sa.Column("meta", sa.JSON(), nullable=True),
-        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
             "source", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
         sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
             "command", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
@@ -51,7 +51,12 @@ def upgrade() -> None:
         sa.Column(
             "output_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
+        sa.Column("owner", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
         sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("source"),
     )
     op.create_table(
         "user_oauth",
@@ -71,6 +76,9 @@ def upgrade() -> None:
         sa.Column(
             "cache_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
         ),
+        sa.Column(
+            "username", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
         sa.PrimaryKeyConstraint("id"),
     )
     op.create_index(
@@ -158,15 +166,15 @@ def upgrade() -> None:
         sa.Column(
             "start_timestamp", sa.DateTime(timezone=True), nullable=True
         ),
-        sa.Column("project_id", sa.Integer(), nullable=False),
-        sa.Column("input_dataset_id", sa.Integer(), nullable=False),
-        sa.Column("output_dataset_id", sa.Integer(), nullable=False),
-        sa.Column("workflow_id", sa.Integer(), nullable=False),
-        sa.Column("overwrite_input", sa.Boolean(), nullable=False),
+        sa.Column("end_timestamp", sa.DateTime(timezone=True), nullable=True),
         sa.Column(
             "worker_init", sqlmodel.sql.sqltypes.AutoString(), nullable=True
         ),
         sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("input_dataset_id", sa.Integer(), nullable=False),
+        sa.Column("output_dataset_id", sa.Integer(), nullable=False),
+        sa.Column("workflow_id", sa.Integer(), nullable=False),
         sa.Column(
             "working_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
         ),

fractal_server/migrations/versions/{fda995215ae9_drop_applyworkflow_overwrite_input.py → f384e1c0cf5d_drop_task_default_args_columns.py}

@@ -1,31 +1,30 @@
-"""Drop ApplyWorkflow.overwrite_input
+"""Drop task.default_args columns
 
-Revision ID: fda995215ae9
-Revises: e8f4051440be
-Create Date: 2023-05-11 17:02:46.208476
+Revision ID: f384e1c0cf5d
+Revises: 4c308bcaea2b
+Create Date: 2023-06-06 15:10:51.838607
 
 """
 import sqlalchemy as sa
 from alembic import op
-
+from sqlalchemy.dialects import sqlite
 
 # revision identifiers, used by Alembic.
-revision = "fda995215ae9"
-down_revision = "e8f4051440be"
+revision = "f384e1c0cf5d"
+down_revision = "4c308bcaea2b"
 branch_labels = None
 depends_on = None
 
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_column("applyworkflow", "overwrite_input")
+    op.drop_column("task", "default_args")
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.add_column(
-        "applyworkflow",
-        sa.Column("overwrite_input", sa.BOOLEAN(), nullable=False),
+        "task", sa.Column("default_args", sqlite.JSON(), nullable=True)
     )
     # ### end Alembic commands ###

fractal_server/tasks/collection.py

@@ -11,14 +11,12 @@
 """
 This module takes care of installing tasks so that fractal can execute them
 
-Tasks can be private or public. Private tasks are installed under
-`Settings.FRACTAL_TASKS_DIR/{username}`. For public tasks, `username =
-.fractal`.
+Tasks are installed under `Settings.FRACTAL_TASKS_DIR/{username}`, with
+`username = ".fractal"`.
 """
 import json
 import shutil
 import sys
-from io import IOBase
 from pathlib import Path
 from typing import Optional
 from typing import Union
@@ -110,11 +108,19 @@ class _TaskCollectPip(TaskCollectPip):
     """
     Internal TaskCollectPip schema
 
-    The difference with its parent class is that we check if the package
-    corresponds to a path in the filesystem, and whether it exists.
+    Differences with its parent class (`TaskCollectPip`):
+
+        1. We check if the package corresponds to a path in the filesystem, and
+           whether it exists (via new validator `check_local_package`, new
+           method `is_local_package` and new attribute `package_path`).
+        2. We include an additional `package_manifest` attribute.
+        3. We expose an additional attribute `package_name`, which is filled
+           during task collection.
     """
 
+    package_name: Optional[str] = None
     package_path: Optional[Path] = None
+    package_manifest: Optional[ManifestV1] = None
 
     @property
     def is_local_package(self) -> bool:
@@ -145,47 +151,64 @@ class _TaskCollectPip(TaskCollectPip):
         return values
 
     @property
-    def pip_package_version(self):
-        """
-        Return pip compatible specification of package and version
-        """
-        version = f"=={self.version}" if self.version else ""
-        return f"{self.package}{version}"
-
-    @property
-    def source(self):
+    def package_source(self):
+        if not self.package_name or not self.package_version:
+            raise ValueError(
+                "Cannot construct `package_source` property with "
+                f"{self.package_name=} and {self.package_version=}."
+            )
         if self.is_local_package:
-            return f"pip-local:{self.package_path.name}"
+            collection_type = "pip_local"
         else:
-            return f"pip:{self.pip_package_version}"
-
-    def check(self):
-        if not self.version:
-            raise ValueError("Version is not set or cannot be determined")
+            collection_type = "pip_remote"
 
+        package_extras = self.package_extras or ""
+        if self.python_version:
+            python_version = f"py{self.python_version}"
+        else:
+            python_version = ""  # FIXME: can we allow this?
+
+        source = ":".join(
+            (
+                collection_type,
+                self.package_name,
+                self.package_version,
+                package_extras,
+                python_version,
+            )
+        )
+        return source
 
-def _package_from_path(wheel_path: Path) -> tuple[str, str]:
-    """
-    Extract package name and version from package files such as wheel files.
-    """
-    wheel_filename = wheel_path.name
-    package, version, *_rest = wheel_filename.split("-")
-    return package, version
+    def check(self):
+        """
+        Verify that the package has all attributes that are needed to continue
+        with task collection
+        """
+        if not self.package_name:
+            raise ValueError("`package_name` attribute is not set")
+        if not self.package_version:
+            raise ValueError("`package_version` attribute is not set")
+        if not self.package_manifest:
+            raise ValueError("`package_manifest` attribute is not set")
 
 
 def create_package_dir_pip(
     *,
     task_pkg: _TaskCollectPip,
-    user: Optional[str] = None,
     create: bool = True,
-    **_,  # FIXME remove this catch-all argument
 ) -> Path:
+    """
+    Create venv folder for a task package and return corresponding Path object
+    """
     settings = Inject(get_settings)
-    user = user or FRACTAL_PUBLIC_TASK_SUBDIR
-
-    package_dir = f"{task_pkg.package}{task_pkg.version or ''}"
+    user = FRACTAL_PUBLIC_TASK_SUBDIR
+    if task_pkg.package_version is None:
+        raise ValueError(
+            f"Cannot create venv folder for package `{task_pkg.package}` "
+            "with `version=None`."
+        )
+    package_dir = f"{task_pkg.package}{task_pkg.package_version}"
     venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir  # type: ignore
-    # TODO check the access right of the venv_path and subdirs
     if create:
         venv_path.mkdir(exist_ok=False, parents=True)
     return venv_path
@@ -201,10 +224,11 @@ async def download_package(
     """
     interpreter = get_python_interpreter(version=task_pkg.python_version)
     pip = f"{interpreter} -m pip"
-    cmd = (
-        f"{pip} download --no-deps {task_pkg.pip_package_version} "
-        f"-d {dest}"
+    version = (
+        f"=={task_pkg.package_version}" if task_pkg.package_version else ""
     )
+    package_and_version = f"{task_pkg.package}{version}"
+    cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
     stdout = await execute_command(command=cmd, cwd=Path("."))
     pkg_file = next(
         line.split()[-1] for line in stdout.split("\n") if "Saved" in line
@@ -212,9 +236,40 @@ async def download_package(
     return Path(pkg_file)
 
 
-def inspect_package(path: Path) -> dict:
+def _load_manifest_from_wheel(
+    path: Path, wheel: ZipFile, logger_name: Optional[str] = None
+) -> ManifestV1:
+    logger = get_logger(logger_name)
+    namelist = wheel.namelist()
+    try:
+        manifest = next(
+            name for name in namelist if "__FRACTAL_MANIFEST__.json" in name
+        )
+    except StopIteration:
+        msg = f"{path.as_posix()} does not include __FRACTAL_MANIFEST__.json"
+        logger.error(msg)
+        raise ValueError(msg)
+    with wheel.open(manifest) as manifest_fd:
+        manifest_dict = json.load(manifest_fd)
+    manifest_version = str(manifest_dict["manifest_version"])
+    if manifest_version == "1":
+        pkg_manifest = ManifestV1(**manifest_dict)
+        return pkg_manifest
+    else:
+        msg = f"Manifest version {manifest_version=} not supported"
+        logger.error(msg)
+        raise ValueError(msg)
+
+
+def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
     """
-    Inspect task package for version and manifest
+    Inspect task package to extract version, name and manifest
+
+    Note that this only works with wheel files, which have a well-defined
+    dist-info section. If we need to generalize to to tar.gz archives, we would
+    need to go and look for `PKG-INFO`.
+
+    Note: package name is normalized by replacing `{-,.}` with `_`.
 
     Args:
         path: Path
@@ -225,32 +280,54 @@ def inspect_package(path: Path) -> dict:
         pacakge, and `manifest`, the Fractal manifest object relative to the
         tasks.
     """
-    if "whl" in path.as_posix():
-        # it is simply a zip file
-        # we can extract the version number from *.dist-info/METADATA
-        # and read the fractal manifest from the package content
-        with ZipFile(path) as wheel:
-            namelist = wheel.namelist()
-            metadata = next(
-                name for name in namelist if "dist-info/METADATA" in name
+
+    logger = get_logger(logger_name)
+
+    if not path.as_posix().endswith(".whl"):
+        raise ValueError(
+            f"Only wheel packages are supported, given {path.as_posix()}."
+        )
+
+    with ZipFile(path) as wheel:
+        namelist = wheel.namelist()
+
+        # Read and validate task manifest
+        logger.debug(f"Now reading manifest for {path.as_posix()}")
+        pkg_manifest = _load_manifest_from_wheel(
+            path, wheel, logger_name=logger_name
+        )
+        logger.debug("Manifest read correctly.")
+
+        # Read package name and version from *.dist-info/METADATA
+        logger.debug(
+            f"Now reading package name and version for {path.as_posix()}"
+        )
+        metadata = next(
+            name for name in namelist if "dist-info/METADATA" in name
+        )
+        with wheel.open(metadata) as metadata_fd:
+            meta = metadata_fd.read().decode("utf-8")
+            pkg_name = next(
+                line.split()[-1]
+                for line in meta.splitlines()
+                if line.startswith("Name")
             )
-            manifest = next(
-                name for name in namelist if "__FRACTAL_MANIFEST__" in name
+            pkg_version = next(
+                line.split()[-1]
+                for line in meta.splitlines()
+                if line.startswith("Version")
             )
+        logger.debug("Package name and version read correctly.")
 
-            with wheel.open(metadata) as metadata_fd:
-                meta = metadata_fd.read().decode("utf-8")
-                version = next(
-                    line.split()[-1]
-                    for line in meta.splitlines()
-                    if line.startswith("Version")
-                )
-
-            with wheel.open(manifest) as manifest_fd:
-                manifest_obj = read_manifest(manifest_fd)  # type: ignore
+    # Normalize package name:
+    pkg_name = pkg_name.replace("-", "_").replace(".", "_")
 
-    version_manifest = dict(version=version, manifest=manifest_obj)
-    return version_manifest
+    info = dict(
+        pkg_name=pkg_name,
+        pkg_version=pkg_version,
+        pkg_manifest=pkg_manifest,
+    )
+    return info
 
 
 async def create_package_environment_pip(
@@ -260,74 +337,60 @@ async def create_package_environment_pip(
     logger_name: str,
 ) -> list[TaskCreate]:
     """
-    Create environment and install package
+    Create environment, install package, and prepare task list
     """
+
     logger = get_logger(logger_name)
+
+    # Only proceed if package, version and manifest attributes are set
+    task_pkg.check()
+
     try:
-        logger.debug("Creating venv and installing package")
 
+        logger.debug("Creating venv and installing package")
         python_bin, package_root = await _create_venv_install_package(
             path=venv_path,
             task_pkg=task_pkg,
             logger_name=logger_name,
         )
-
-        logger.debug("Loading task manifest")
-
-        task_list = load_manifest(
-            package_root=package_root,
-            python_bin=python_bin,
-            source=task_pkg.source,
-        )
-        logger.debug("Loaded task manifest")
+        logger.debug("Venv creation and package installation ended correctly.")
+
+        # Prepare task_list with appropriate metadata
+        logger.debug("Creating task list from manifest")
+        task_list = []
+        for t in task_pkg.package_manifest.task_list:
+            # Fill in attributes for TaskCreate
+            task_executable = package_root / t.executable
+            cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
+            task_name_slug = t.name.replace(" ", "_").lower()
+            task_source = f"{task_pkg.package_source}:{task_name_slug}"
+            if not task_executable.exists():
+                raise FileNotFoundError(
+                    f"Cannot find executable `{task_executable}` "
+                    f"for task `{t.name}`"
+                )
+            manifest = task_pkg.package_manifest
+            if manifest.has_args_schemas:
+                additional_attrs = dict(
+                    args_schema_version=manifest.args_schema_version
+                )
+            else:
+                additional_attrs = {}
+            this_task = TaskCreate(
+                **t.dict(),
+                command=cmd,
+                version=task_pkg.package_version,
+                **additional_attrs,
+                source=task_source,
+            )
+            task_list.append(this_task)
+        logger.debug("Task list created correctly")
     except Exception as e:
         logger.error("Task manifest loading failed")
         raise e
     return task_list
 
 
-def read_manifest(file: Union[Path, IOBase]) -> ManifestV1:
-    """
-    Read and parse manifest file
-    """
-    if isinstance(file, IOBase):
-        manifest_dict = json.load(file)
-    else:
-        with file.open("r") as f:
-            manifest_dict = json.load(f)
-
-    manifest_version = str(manifest_dict["manifest_version"])
-    if manifest_version == "1":
-        manifest = ManifestV1(**manifest_dict)
-    else:
-        raise ValueError("Manifest version {manifest_version=} not supported")
-
-    return manifest
-
-
-def load_manifest(
-    package_root: Path,
-    python_bin: Path,
-    source: str,
-) -> list[TaskCreate]:
-
-    manifest_file = package_root / "__FRACTAL_MANIFEST__.json"
-    manifest = read_manifest(manifest_file)
-
-    task_list = []
-    for t in manifest.task_list:
-        task_executable = package_root / t.executable
-        if not task_executable.exists():
-            raise FileNotFoundError(
-                f"Cannot find executable `{task_executable}` "
-                f"for task `{t.name}`"
-            )
-        cmd = f"{python_bin.as_posix()} {task_executable.as_posix()}"
-        this_task = TaskCreate(**t.dict(), command=cmd, source=source)
-        task_list.append(this_task)
-    return task_list
-
-
 async def _create_venv_install_package(
     *,
     task_pkg: _TaskCollectPip,
@@ -403,7 +466,9 @@ async def _pip_install(
     if task_pkg.is_local_package:
         pip_install_str = f"{task_pkg.package_path.as_posix()}{extras}"
     else:
-        version_string = f"=={task_pkg.version}" if task_pkg.version else ""
+        version_string = (
+            f"=={task_pkg.package_version}" if task_pkg.package_version else ""
+        )
         pip_install_str = f"{task_pkg.package}{extras}{version_string}"
 
     cmd_install = f"{pip} install {pip_install_str}"

{fractal_server-1.3.0a2.dist-info → fractal_server-1.3.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 1.3.0a2
+Version: 1.3.0a3
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause
@@ -25,6 +25,7 @@ Requires-Dist: fastapi (>=0.95.0,<0.96.0)
 Requires-Dist: fastapi-users[oauth] (>=10.1,<11.0)
 Requires-Dist: gunicorn (>=20.1.0,<21.0.0) ; extra == "gunicorn"
 Requires-Dist: psycopg2 (>=2.9.5,<3.0.0) ; extra == "postgres"
+Requires-Dist: pydantic (>=1.10.8)
 Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
 Requires-Dist: sqlalchemy (>=1.4,<2.0)
 Requires-Dist: sqlmodel (>=0.0.8,<0.0.9)