fractal-server 2.7.1__py3-none-any.whl → 2.8.0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.7.1"
+__VERSION__ = "2.8.0"

fractal_server/app/models/user_settings.py

@@ -36,3 +36,4 @@ class UserSettings(SQLModel, table=True):
     ssh_jobs_dir: Optional[str] = None
     slurm_user: Optional[str] = None
     cache_dir: Optional[str] = None
+    project_dir: Optional[str] = None

fractal_server/app/models/v2/task.py

@@ -103,3 +103,18 @@ class TaskGroupV2(SQLModel, table=True):
                     f"{self.pkg_name=}, {self.wheel_path=}, {self.version=}."
                 )
             return f"{self.pkg_name}{extras}=={self.version}"
+
+    @property
+    def pinned_package_versions_string(self) -> str:
+        """
+        Prepare string to be used in `python -m pip install`.
+        """
+        if self.pinned_package_versions is None:
+            return ""
+        output = " ".join(
+            [
+                f"{key}=={value}"
+                for key, value in self.pinned_package_versions.items()
+            ]
+        )
+        return output

fractal_server/app/routes/api/v2/dataset.py

@@ -22,6 +22,8 @@ from ._aux_functions import _get_project_check_owner
 from ._aux_functions import _get_submitted_jobs_statement
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
+from fractal_server.string_tools import sanitize_string
+from fractal_server.urls import normalize_url
 
 router = APIRouter()
 
@@ -40,14 +42,45 @@ async def create_dataset(
     """
     Add new dataset to current project
     """
-    await _get_project_check_owner(
+    project = await _get_project_check_owner(
         project_id=project_id, user_id=user.id, db=db
     )
-    db_dataset = DatasetV2(project_id=project_id, **dataset.dict())
-    db.add(db_dataset)
-    await db.commit()
-    await db.refresh(db_dataset)
-    await db.close()
+
+    if dataset.zarr_dir is None:
+
+        if user.settings.project_dir is None:
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    "Both 'dataset.zarr_dir' and 'user.settings.project_dir' "
+                    "are null"
+                ),
+            )
+
+        db_dataset = DatasetV2(
+            project_id=project_id,
+            zarr_dir="__PLACEHOLDER__",
+            **dataset.dict(exclude={"zarr_dir"}),
+        )
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
+        path = (
+            f"{user.settings.project_dir}/fractal/"
+            f"{project_id}_{sanitize_string(project.name)}/"
+            f"{db_dataset.id}_{sanitize_string(db_dataset.name)}"
+        )
+        normalized_path = normalize_url(path)
+        db_dataset.zarr_dir = normalized_path
+
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
+    else:
+        db_dataset = DatasetV2(project_id=project_id, **dataset.dict())
+        db.add(db_dataset)
+        await db.commit()
+        await db.refresh(db_dataset)
 
     return db_dataset
 

fractal_server/app/routes/api/v2/task.py

@@ -36,8 +36,7 @@ logger = set_logger(__name__)
 
 @router.get("/", response_model=list[TaskReadV2])
 async def get_list_task(
-    args_schema_parallel: bool = True,
-    args_schema_non_parallel: bool = True,
+    args_schema: bool = True,
     category: Optional[str] = None,
     modality: Optional[str] = None,
     author: Optional[str] = None,
@@ -72,11 +71,9 @@ async def get_list_task(
     res = await db.execute(stm)
     task_list = res.scalars().all()
     await db.close()
-    if args_schema_parallel is False:
+    if args_schema is False:
         for task in task_list:
             setattr(task, "args_schema_parallel", None)
-    if args_schema_non_parallel is False:
-        for task in task_list:
             setattr(task, "args_schema_non_parallel", None)
 
     return task_list

fractal_server/app/routes/api/v2/task_collection.py

@@ -24,6 +24,7 @@ from ....schemas.v2 import CollectionStatusV2
 from ....schemas.v2 import TaskCollectPipV2
 from ....schemas.v2 import TaskGroupCreateV2
 from ...aux.validate_user_settings import validate_user_settings
+from ._aux_functions_task_collection import get_package_version_from_pypi
 from ._aux_functions_tasks import _get_valid_user_group_id
 from ._aux_functions_tasks import _verify_non_duplication_group_constraint
 from ._aux_functions_tasks import _verify_non_duplication_user_constraint
@@ -31,16 +32,14 @@ from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.auth import current_active_verified_user
 from fractal_server.app.schemas.v2 import TaskGroupV2OriginEnum
-from fractal_server.tasks.utils import _normalize_package_name
-from fractal_server.tasks.utils import get_collection_log_v2
-from fractal_server.tasks.v2.background_operations import (
-    background_collect_pip,
+from fractal_server.tasks.v2.collection_local import (
+    collect_package_local,
 )
-from fractal_server.tasks.v2.endpoint_operations import (
-    get_package_version_from_pypi,
+from fractal_server.tasks.v2.utils_package_names import _parse_wheel_filename
+from fractal_server.tasks.v2.utils_package_names import normalize_package_name
+from fractal_server.tasks.v2.utils_python_interpreter import (
+    get_python_interpreter_v2,
 )
-from fractal_server.tasks.v2.utils import _parse_wheel_filename
-from fractal_server.tasks.v2.utils import get_python_interpreter_v2
 
 router = APIRouter()
 
@@ -118,14 +117,14 @@ async def collect_tasks_pip(
                     f"Original error: {str(e)}",
                 ),
             )
-        task_group_attrs["pkg_name"] = _normalize_package_name(
+        task_group_attrs["pkg_name"] = normalize_package_name(
             wheel_info["distribution"]
         )
         task_group_attrs["version"] = wheel_info["version"]
         task_group_attrs["origin"] = TaskGroupV2OriginEnum.WHEELFILE
     else:
         pkg_name = task_collect.package
-        task_group_attrs["pkg_name"] = _normalize_package_name(pkg_name)
+        task_group_attrs["pkg_name"] = normalize_package_name(pkg_name)
         task_group_attrs["origin"] = TaskGroupV2OriginEnum.PYPI
         latest_version = await get_package_version_from_pypi(
             task_collect.package,
@@ -249,8 +248,8 @@ async def collect_tasks_pip(
     if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
         # SSH task collection
 
-        from fractal_server.tasks.v2.background_operations_ssh import (
-            background_collect_pip_ssh,
+        from fractal_server.tasks.v2.collection_ssh import (
+            collect_package_ssh,
         )
 
         # User appropriate FractalSSH object
@@ -263,7 +262,7 @@ async def collect_tasks_pip(
         fractal_ssh = fractal_ssh_list.get(**ssh_credentials)
 
         background_tasks.add_task(
-            background_collect_pip_ssh,
+            collect_package_ssh,
             state_id=state.id,
             task_group=task_group,
             fractal_ssh=fractal_ssh,
@@ -273,7 +272,7 @@ async def collect_tasks_pip(
     else:
         # Local task collection
         background_tasks.add_task(
-            background_collect_pip,
+            collect_package_local,
             state_id=state.id,
             task_group=task_group,
         )
@@ -296,42 +295,15 @@ async def collect_tasks_pip(
 async def check_collection_status(
     state_id: int,
     user: UserOAuth = Depends(current_active_user),
-    verbose: bool = False,
     db: AsyncSession = Depends(get_async_db),
 ) -> CollectionStateReadV2:  # State[TaskCollectStatus]
     """
     Check status of background task collection
     """
-
-    logger = set_logger(logger_name="check_collection_status")
-    logger.debug(f"Querying state for state.id={state_id}")
     state = await db.get(CollectionStateV2, state_id)
-    if not state:
-        await db.close()
+    if state is None:
         raise HTTPException(
             status_code=status.HTTP_404_NOT_FOUND,
             detail=f"No task collection info with id={state_id}",
         )
-
-    settings = Inject(get_settings)
-    if settings.FRACTAL_RUNNER_BACKEND == "slurm_ssh":
-        # FIXME SSH: add logic for when data.state["log"] is empty
-        pass
-    else:
-        # Non-SSH mode
-        # In some cases (i.e. a successful or ongoing task collection),
-        # state.data["log"] is not set; if so, we collect the current logs.
-        if verbose and not state.data.get("log"):
-            if "path" not in state.data.keys():
-                await db.close()
-                raise HTTPException(
-                    status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-                    detail=(
-                        f"No 'path' in CollectionStateV2[{state_id}].data"
-                    ),
-                )
-            state.data["log"] = get_collection_log_v2(Path(state.data["path"]))
-
-    reset_logger_handlers(logger)
-    await db.close()
     return state

fractal_server/app/routes/api/v2/task_collection_custom.py

@@ -27,12 +27,12 @@ from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.string_tools import validate_cmd
 from fractal_server.syringe import Inject
-from fractal_server.tasks.v2.background_operations import (
-    _prepare_tasks_metadata,
-)
 from fractal_server.tasks.v2.database_operations import (
     create_db_tasks_and_update_task_group,
 )
+from fractal_server.tasks.v2.utils_background import (
+    _prepare_tasks_metadata,
+)
 
 router = APIRouter()
 

fractal_server/app/schemas/user_settings.py

@@ -19,6 +19,10 @@ __all__ = (
 
 
 class UserSettingsRead(BaseModel):
+    """
+    Schema reserved for superusers
+    """
+
     id: int
     ssh_host: Optional[str] = None
     ssh_username: Optional[str] = None
@@ -28,6 +32,7 @@ class UserSettingsRead(BaseModel):
     slurm_user: Optional[str] = None
     slurm_accounts: list[str]
     cache_dir: Optional[str] = None
+    project_dir: Optional[str] = None
 
 
 class UserSettingsReadStrict(BaseModel):
@@ -35,9 +40,14 @@ class UserSettingsReadStrict(BaseModel):
     slurm_accounts: list[str]
     cache_dir: Optional[str] = None
     ssh_username: Optional[str] = None
+    project_dir: Optional[str] = None
 
 
 class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
+    """
+    Schema reserved for superusers
+    """
+
     ssh_host: Optional[str] = None
     ssh_username: Optional[str] = None
     ssh_private_key_path: Optional[str] = None
@@ -46,6 +56,7 @@ class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
     slurm_user: Optional[str] = None
     slurm_accounts: Optional[list[StrictStr]] = None
     cache_dir: Optional[str] = None
+    project_dir: Optional[str] = None
 
     _ssh_host = validator("ssh_host", allow_reuse=True)(
         valstr("ssh_host", accept_none=True)
@@ -83,6 +94,13 @@ class UserSettingsUpdate(BaseModel, extra=Extra.forbid):
         validate_cmd(value)
         return val_absolute_path("cache_dir")(value)
 
+    @validator("project_dir")
+    def project_dir_validator(cls, value):
+        if value is None:
+            return None
+        validate_cmd(value)
+        return val_absolute_path("project_dir")(value)
+
 
 class UserSettingsUpdateStrict(BaseModel, extra=Extra.forbid):
     slurm_accounts: Optional[list[StrictStr]] = None

fractal_server/app/schemas/v2/dataset.py

@@ -33,14 +33,16 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
 
-    zarr_dir: str
+    zarr_dir: Optional[str] = None
 
     filters: Filters = Field(default_factory=Filters)
 
     # Validators
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: str) -> str:
-        return normalize_url(v)
+        if v is not None:
+            return normalize_url(v)
+        return v
 
     _name = validator("name", allow_reuse=True)(valstr("name"))
 
@@ -95,7 +97,7 @@ class DatasetImportV2(BaseModel, extra=Extra.forbid):
 
     name: str
     zarr_dir: str
-    images: list[SingleImage] = Field(default_factory=[])
+    images: list[SingleImage] = Field(default_factory=list)
     filters: Filters = Field(default_factory=Filters)
 
     # Validators

fractal_server/app/schemas/v2/task_collection.py

@@ -10,7 +10,6 @@ from pydantic import Extra
 from pydantic import root_validator
 from pydantic import validator
 
-from .._validators import valdictkeys
 from .._validators import valstr
 from fractal_server.app.schemas._validators import valutc
 from fractal_server.app.schemas.v2 import ManifestV2
@@ -62,13 +61,30 @@ class TaskCollectPipV2(BaseModel, extra=Extra.forbid):
     _package_version = validator("package_version", allow_reuse=True)(
         valstr("package_version")
     )
-    _pinned_package_versions = validator(
-        "pinned_package_versions", allow_reuse=True
-    )(valdictkeys("pinned_package_versions"))
     _package_extras = validator("package_extras", allow_reuse=True)(
         valstr("package_extras")
     )
 
+    @validator("pinned_package_versions")
+    def pinned_package_validator(cls, value):
+        if value is None:
+            return value
+        old_keys = list(value.keys())
+        new_keys = [
+            valstr(f"pinned_package_versions[{key}]")(key) for key in old_keys
+        ]
+        if len(new_keys) != len(set(new_keys)):
+            raise ValueError(
+                f"Dictionary contains multiple identical keys: {value}."
+            )
+        for old_key, new_key in zip(old_keys, new_keys):
+            if new_key != old_key:
+                value[new_key] = value.pop(old_key)
+        for pkg, version in value.items():
+            validate_cmd(pkg)
+            validate_cmd(version)
+        return value
+
     @validator("package")
     def package_validator(cls, value):
         if "/" in value or value.endswith(".whl"):

fractal_server/migrations/versions/19eca0dd47a9_user_settings_project_dir.py

@@ -0,0 +1,39 @@
+"""user settings project dir
+
+Revision ID: 19eca0dd47a9
+Revises: 8e8f227a3e36
+Create Date: 2024-10-30 14:34:28.219355
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "19eca0dd47a9"
+down_revision = "8e8f227a3e36"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "project_dir",
+                sqlmodel.sql.sqltypes.AutoString(),
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("user_settings", schema=None) as batch_op:
+        batch_op.drop_column("project_dir")
+
+    # ### end Alembic commands ###

fractal_server/tasks/utils.py

@@ -1,4 +1,3 @@
-import re
 from pathlib import Path
 
 from fractal_server.config import get_settings
@@ -42,38 +41,8 @@ def get_collection_log_v1(path: Path) -> str:
     return log
 
 
-def get_collection_log_v2(path: Path) -> str:
-    log_path = get_log_path(path)
-    log = log_path.open().read()
-    return log
-
-
 def get_collection_freeze_v1(venv_path: Path) -> str:
     package_path = get_absolute_venv_path_v1(venv_path)
     freeze_path = get_freeze_path(package_path)
     freeze = freeze_path.open().read()
     return freeze
-
-
-def get_collection_freeze_v2(path: Path) -> str:
-    freeze_path = get_freeze_path(path)
-    freeze = freeze_path.open().read()
-    return freeze
-
-
-def _normalize_package_name(name: str) -> str:
-    """
-    Implement PyPa specifications for package-name normalization
-
-    The name should be lowercased with all runs of the characters `.`, `-`,
-    or `_` replaced with a single `-` character. This can be implemented in
-    Python with the re module.
-    (https://packaging.python.org/en/latest/specifications/name-normalization)
-
-    Args:
-        name: The non-normalized package name.
-
-    Returns:
-        The normalized package name.
-    """
-    return re.sub(r"[-_.]+", "-", name).lower()

fractal_server/tasks/v1/background_operations.py

@@ -7,10 +7,10 @@ from pathlib import Path
 from shutil import rmtree as shell_rmtree
 
 from ...string_tools import slugify_task_name_for_source_v1
-from ..utils import _normalize_package_name
 from ..utils import get_collection_log_v1
 from ..utils import get_collection_path
 from ..utils import get_log_path
+from ..v2.utils_package_names import normalize_package_name
 from ._TaskCollectPip import _TaskCollectPip
 from .utils import _init_venv_v1
 from fractal_server.app.db import DBSyncSession
@@ -23,7 +23,7 @@ from fractal_server.app.schemas.v1 import TaskReadV1
 from fractal_server.logger import close_logger
 from fractal_server.logger import get_logger
 from fractal_server.logger import set_logger
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 async def _pip_install(
@@ -60,12 +60,12 @@ async def _pip_install(
     cmd_install = f"{pip} install {pip_install_str}"
     cmd_inspect = f"{pip} show {task_pkg.package}"
 
-    await execute_command(
+    await execute_command_async(
         cwd=venv_path,
         command=f"{pip} install --upgrade pip",
         logger_name=logger_name,
     )
-    await execute_command(
+    await execute_command_async(
         cwd=venv_path, command=cmd_install, logger_name=logger_name
     )
     if task_pkg.pinned_package_versions:
@@ -82,7 +82,7 @@ async def _pip_install(
                 "Preliminary check: verify that "
                 f"{pinned_pkg_version} is already installed"
             )
-            stdout_inspect = await execute_command(
+            stdout_inspect = await execute_command_async(
                 cwd=venv_path,
                 command=f"{pip} show {pinned_pkg_name}",
                 logger_name=logger_name,
@@ -99,7 +99,7 @@ async def _pip_install(
                     f"({pinned_pkg_version}); "
                     f"install version {pinned_pkg_version}."
                 )
-                await execute_command(
+                await execute_command_async(
                     cwd=venv_path,
                     command=(
                         f"{pip} install "
@@ -114,7 +114,7 @@ async def _pip_install(
                 )
 
     # Extract package installation path from `pip show`
-    stdout_inspect = await execute_command(
+    stdout_inspect = await execute_command_async(
         cwd=venv_path, command=cmd_inspect, logger_name=logger_name
     )
 
@@ -165,8 +165,8 @@ async def _create_venv_install_package(
     """
 
     # Normalize package name
-    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
-    task_pkg.package = _normalize_package_name(task_pkg.package)
+    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
+    task_pkg.package = normalize_package_name(task_pkg.package)
 
     python_bin = await _init_venv_v1(
         path=path,
@@ -192,8 +192,8 @@ async def create_package_environment_pip(
     logger = get_logger(logger_name)
 
     # Normalize package name
-    task_pkg.package_name = _normalize_package_name(task_pkg.package_name)
-    task_pkg.package = _normalize_package_name(task_pkg.package)
+    task_pkg.package_name = normalize_package_name(task_pkg.package_name)
+    task_pkg.package = normalize_package_name(task_pkg.package)
 
     # Only proceed if package, version and manifest attributes are set
     task_pkg.check()

fractal_server/tasks/v1/endpoint_operations.py

@@ -4,14 +4,14 @@ from typing import Optional
 from typing import Union
 from zipfile import ZipFile
 
-from ..utils import _normalize_package_name
+from ..v2.utils_package_names import normalize_package_name
 from ._TaskCollectPip import _TaskCollectPip as _TaskCollectPipV1
 from .utils import get_python_interpreter_v1
 from fractal_server.app.schemas.v1 import ManifestV1
 from fractal_server.config import get_settings
 from fractal_server.logger import get_logger
 from fractal_server.syringe import Inject
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 FRACTAL_PUBLIC_TASK_SUBDIR = ".fractal"
@@ -32,7 +32,7 @@ async def download_package(
     )
     package_and_version = f"{task_pkg.package}{version}"
     cmd = f"{pip} download --no-deps {package_and_version} -d {dest}"
-    stdout = await execute_command(command=cmd, cwd=Path("."))
+    stdout = await execute_command_async(command=cmd, cwd=Path("."))
     pkg_file = next(
         line.split()[-1] for line in stdout.split("\n") if "Saved" in line
     )
@@ -123,7 +123,7 @@ def inspect_package(path: Path, logger_name: Optional[str] = None) -> dict:
         logger.debug("Package name and version read correctly.")
 
     # Normalize package name:
-    pkg_name = _normalize_package_name(pkg_name)
+    pkg_name = normalize_package_name(pkg_name)
 
     info = dict(
         pkg_name=pkg_name,
@@ -148,7 +148,7 @@ def create_package_dir_pip(
             f"Cannot create venv folder for package `{task_pkg.package}` "
             "with `version=None`."
         )
-    normalized_package = _normalize_package_name(task_pkg.package)
+    normalized_package = normalize_package_name(task_pkg.package)
     package_dir = f"{normalized_package}{task_pkg.package_version}"
     venv_path = settings.FRACTAL_TASKS_DIR / user / package_dir
     if create:

fractal_server/tasks/v1/utils.py

@@ -2,7 +2,7 @@ from pathlib import Path
 from typing import Optional
 
 from fractal_server.logger import get_logger
-from fractal_server.utils import execute_command
+from fractal_server.utils import execute_command_async
 
 
 def get_python_interpreter_v1(version: Optional[str] = None) -> str:
@@ -57,7 +57,7 @@ async def _init_venv_v1(
     logger.debug(f"[_init_venv] {path=}")
     interpreter = get_python_interpreter_v1(version=python_version)
     logger.debug(f"[_init_venv] {interpreter=}")
-    await execute_command(
+    await execute_command_async(
         cwd=path,
         command=f"{interpreter} -m venv venv",
         logger_name=logger_name,