fractal-server 2.14.3__tar.gz → 2.14.4a0__tar.gz

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.14.3
+Version: 2.14.4a0
 Summary: Backend component of the Fractal analytics platform
 License: BSD-3-Clause
 Author: Tommaso Comparin

fractal_server-2.14.4a0/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.14.4a0"

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/models/v2/job.py

@@ -11,7 +11,7 @@ from sqlmodel import SQLModel
 
 from ....utils import get_timestamp
 from ...schemas.v2 import JobStatusTypeV2
-from fractal_server.images.models import AttributeFiltersType
+from fractal_server.types import AttributeFilters
 
 
 class JobV2(SQLModel, table=True):
@@ -57,7 +57,7 @@ class JobV2(SQLModel, table=True):
     status: str = JobStatusTypeV2.SUBMITTED
     log: Optional[str] = None
 
-    attribute_filters: AttributeFiltersType = Field(
+    attribute_filters: AttributeFilters = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )
     type_filters: dict[str, bool] = Field(

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/routes/api/v2/images.py

@@ -8,8 +8,6 @@ from fastapi import Response
 from fastapi import status
 from pydantic import BaseModel
 from pydantic import Field
-from pydantic import field_validator
-from pydantic import model_validator
 from sqlalchemy.orm.attributes import flag_modified
 from sqlmodel import delete
 
@@ -22,18 +20,14 @@ from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.routes.pagination import get_pagination_params
 from fractal_server.app.routes.pagination import PaginationRequest
 from fractal_server.app.routes.pagination import PaginationResponse
-from fractal_server.app.schemas._filter_validators import (
-    validate_attribute_filters,
-)
-from fractal_server.app.schemas._filter_validators import validate_type_filters
-from fractal_server.app.schemas._validators import root_validate_dict_keys
 from fractal_server.images import SingleImage
 from fractal_server.images import SingleImageUpdate
-from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import aggregate_attributes
 from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import find_image_by_zarr_url
 from fractal_server.images.tools import match_filter
+from fractal_server.types import AttributeFilters
+from fractal_server.types import TypeFilters
 
 router = APIRouter()
 
@@ -45,18 +39,8 @@ class ImagePage(PaginationResponse[SingleImage]):
 
 
 class ImageQuery(BaseModel):
-    type_filters: dict[str, bool] = Field(default_factory=dict)
-    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
-
-    _dict_keys = model_validator(mode="before")(
-        classmethod(root_validate_dict_keys)
-    )
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
+    type_filters: TypeFilters = Field(default_factory=dict)
+    attribute_filters: AttributeFilters = Field(default_factory=dict)
 
 
 class ImageQueryWithZarrUrl(ImageQuery):

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/routes/api/v2/pre_submission_checks.py

@@ -18,9 +18,9 @@ from fractal_server.app.models.v2 import HistoryImageCache
 from fractal_server.app.models.v2 import HistoryUnit
 from fractal_server.app.routes.auth import current_active_user
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
-from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import aggregate_types
 from fractal_server.images.tools import filter_image_list
+from fractal_server.types import AttributeFilters
 
 router = APIRouter()
 
@@ -71,7 +71,7 @@ async def verify_unique_types(
 
 
 class NonProcessedImagesPayload(BaseModel):
-    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+    attribute_filters: AttributeFilters = Field(default_factory=dict)
     type_filters: dict[str, bool] = Field(default_factory=dict)
 
 

{fractal_server-2.14.3/fractal_server/app/runner → fractal_server-2.14.4a0/fractal_server/app/runner/executors/slurm_ssh}/run_subprocess.py

@@ -15,17 +15,22 @@ def run_subprocess(
     logger = get_logger(logger_name)
     try:
         res = subprocess.run(  # nosec
-            shlex.split(cmd), check=True, capture_output=True, encoding="utf-8"
+            shlex.split(cmd),
+            check=True,
+            capture_output=True,
+            encoding="utf-8",
         )
         return res
     except subprocess.CalledProcessError as e:
-        logger.debug(
+        logger.info(
             f"Command '{e.cmd}' returned non-zero exit status {e.returncode}."
         )
-        logger.debug(f"stdout: {e.stdout}")
-        logger.debug(f"stderr: {e.stderr}")
+        logger.info(f"stdout: {e.stdout}")
+        logger.info(f"stderr: {e.stderr}")
         raise e
     except Exception as e:
-        logger.debug(f"An error occurred while running command: {cmd}")
-        logger.debug(str(e))
+        logger.warning(
+            f"An unexpected error occurred while running command: {cmd}"
+        )
+        logger.warning(str(e))
         raise e

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/executors/slurm_ssh/runner.py

@@ -4,8 +4,9 @@ from typing import Optional
 
 from ..slurm_common.base_slurm_runner import BaseSlurmRunner
 from ..slurm_common.slurm_job_task_models import SlurmJob
-from fractal_server.app.runner.compress_folder import compress_folder
-from fractal_server.app.runner.extract_archive import extract_archive
+from .run_subprocess import run_subprocess
+from .tar_commands import get_tar_compression_cmd
+from .tar_commands import get_tar_extraction_cmd
 from fractal_server.config import get_settings
 from fractal_server.logger import set_logger
 from fractal_server.ssh._fabric import FractalSSH
@@ -84,12 +85,8 @@ class SlurmSSHRunner(BaseSlurmRunner):
         workdir_remote = finished_slurm_jobs[0].workdir_remote
 
         # Define local/remote tarfile paths
-        tarfile_path_local = (
-            workdir_local.parent / f"{workdir_local.name}.tar.gz"
-        ).as_posix()
-        tarfile_path_remote = (
-            workdir_remote.parent / f"{workdir_remote.name}.tar.gz"
-        ).as_posix()
+        tarfile_path_local = workdir_local.with_suffix(".tar.gz").as_posix()
+        tarfile_path_remote = workdir_remote.with_suffix(".tar.gz").as_posix()
 
         # Create file list
         # NOTE: see issue 2483
@@ -130,11 +127,9 @@ class SlurmSSHRunner(BaseSlurmRunner):
 
         # Create remote tarfile
         t_0_tar = time.perf_counter()
-        tar_command = (
-            f"{self.python_worker_interpreter} "
-            "-m fractal_server.app.runner.compress_folder "
-            f"{workdir_remote.as_posix()} "
-            f"--filelist {tmp_filelist_path}"
+        tar_command = get_tar_compression_cmd(
+            subfolder_path=workdir_remote,
+            filelist_path=tmp_filelist_path,
         )
         self.fractal_ssh.run_command(cmd=tar_command)
         t_1_tar = time.perf_counter()
@@ -157,54 +152,14 @@ class SlurmSSHRunner(BaseSlurmRunner):
         )
 
         # Extract tarfile locally
-        extract_archive(Path(tarfile_path_local))
-
-        # Remove local tarfile
+        target_dir, cmd_tar = get_tar_extraction_cmd(Path(tarfile_path_local))
+        target_dir.mkdir(exist_ok=True)
+        run_subprocess(cmd=cmd_tar, logger_name=logger.name)
         Path(tarfile_path_local).unlink(missing_ok=True)
 
         t_1 = time.perf_counter()
         logger.info(f"[_fetch_artifacts] End - elapsed={t_1 - t_0:.3f} s")
 
-    def _send_inputs(self, jobs: list[SlurmJob]) -> None:
-        """
-        Transfer the jobs subfolder to the remote host.
-        """
-        for job in jobs:
-            # Create local archive
-            tarfile_path_local = compress_folder(
-                job.workdir_local,
-                filelist_path=None,
-            )
-            tarfile_name = Path(tarfile_path_local).name
-            logger.info(f"Subfolder archive created at {tarfile_path_local}")
-
-            # Transfer archive
-            tarfile_path_remote = (
-                job.workdir_remote.parent / tarfile_name
-            ).as_posix()
-            t_0_put = time.perf_counter()
-            self.fractal_ssh.send_file(
-                local=tarfile_path_local,
-                remote=tarfile_path_remote,
-            )
-            t_1_put = time.perf_counter()
-            logger.info(
-                f"Subfolder archive transferred to {tarfile_path_remote}"
-                f" - elapsed={t_1_put - t_0_put:.3f} s"
-            )
-
-            # Remove local archive
-            Path(tarfile_path_local).unlink()
-            logger.debug(f"Local archive {tarfile_path_local} removed")
-
-            # Uncompress remote archive
-            tar_command = (
-                f"{self.python_worker_interpreter} -m "
-                "fractal_server.app.runner.extract_archive "
-                f"{tarfile_path_remote}"
-            )
-            self.fractal_ssh.run_command(cmd=tar_command)
-
     def _run_remote_cmd(self, cmd: str) -> str:
         stdout = self.fractal_ssh.run_command(cmd=cmd)
         return stdout

fractal_server-2.14.4a0/fractal_server/app/runner/executors/slurm_ssh/tar_commands.py

@@ -0,0 +1,65 @@
+"""
+Prepare tar commands for task-subfolder compression/extraction.
+"""
+from pathlib import Path
+
+
+def get_tar_compression_cmd(
+    subfolder_path: Path,
+    filelist_path: Path | None,
+) -> tuple[str, str]:
+    """
+    Prepare command to compress e.g. `/path/dir` into `/path/dir.tar.gz`.
+
+    Note that `/path/dir.tar.gz` may already exist. In this case, it will
+    be overwritten.
+
+    Args:
+        subfolder_path: Absolute path to the folder to compress.
+        filelist_path: If set, to be used in the `--files-from` option.
+        expected_tarfile: If set, it should match to the returned one.
+
+    Returns:
+        tar command
+    """
+    tarfile_path = subfolder_path.with_suffix(".tar.gz")
+    if filelist_path is None:
+        cmd_tar = (
+            f"tar -c -z "
+            f"-f {tarfile_path} "
+            f"--directory={subfolder_path.as_posix()} "
+            "."
+        )
+    else:
+        cmd_tar = (
+            f"tar -c -z -f {tarfile_path} "
+            f"--directory={subfolder_path.as_posix()} "
+            f"--files-from={filelist_path.as_posix()} --ignore-failed-read"
+        )
+
+    return cmd_tar
+
+
+def get_tar_extraction_cmd(archive_path: Path) -> tuple[Path, str]:
+    """
+    Prepare command to extract e.g. `/path/dir.tar.gz` into `/path/dir`.
+
+    Args:
+        archive_path: Absolute path to the tar.gz archive.
+
+    Returns:
+        Target extraction folder and tar command
+    """
+
+    # Prepare subfolder path
+    if archive_path.suffixes[-2:] != [".tar", ".gz"]:
+        raise ValueError(
+            "Archive path must end with `.tar.gz` "
+            f"(given: {archive_path.as_posix()})"
+        )
+    subfolder_path = archive_path.with_suffix("").with_suffix("")
+
+    cmd_tar = (
+        f"tar -xzvf {archive_path} --directory={subfolder_path.as_posix()}"
+    )
+    return subfolder_path, cmd_tar

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/v2/_local.py

@@ -7,7 +7,7 @@ from ..executors.local.get_local_config import get_local_backend_config
 from ..executors.local.runner import LocalRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
-from fractal_server.images.models import AttributeFiltersType
+from fractal_server.types import AttributeFilters
 
 
 def process_workflow(
@@ -20,7 +20,7 @@ def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
+    job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
     **kwargs,

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/v2/_slurm_ssh.py

@@ -27,8 +27,8 @@ from ..executors.slurm_common.get_slurm_config import get_slurm_config
 from ..executors.slurm_ssh.runner import SlurmSSHRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
-from fractal_server.images.models import AttributeFiltersType
 from fractal_server.logger import set_logger
+from fractal_server.types import AttributeFilters
 
 logger = set_logger(__name__)
 
@@ -43,7 +43,7 @@ def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
+    job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     fractal_ssh: FractalSSH,
     worker_init: Optional[str] = None,

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/v2/_slurm_sudo.py

@@ -25,7 +25,7 @@ from ..executors.slurm_common.get_slurm_config import get_slurm_config
 from ..executors.slurm_sudo.runner import SudoSlurmRunner
 from ..set_start_and_last_task_index import set_start_and_last_task_index
 from .runner import execute_tasks_v2
-from fractal_server.images.models import AttributeFiltersType
+from fractal_server.types import AttributeFilters
 
 
 def process_workflow(
@@ -38,7 +38,7 @@ def process_workflow(
     first_task_index: Optional[int] = None,
     last_task_index: Optional[int] = None,
     logger_name: str,
-    job_attribute_filters: AttributeFiltersType,
+    job_attribute_filters: AttributeFilters,
     job_type_filters: dict[str, bool],
     user_id: int,
     # Slurm-specific

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/v2/runner.py

@@ -34,8 +34,8 @@ from fractal_server.app.runner.v2.db_tools import update_status_of_history_run
 from fractal_server.app.schemas.v2 import HistoryUnitStatus
 from fractal_server.app.schemas.v2 import TaskDumpV2
 from fractal_server.app.schemas.v2 import TaskGroupDumpV2
-from fractal_server.images.models import AttributeFiltersType
 from fractal_server.images.tools import merge_type_filters
+from fractal_server.types import AttributeFilters
 
 
 def drop_none_attributes(attributes: dict[str, Any]) -> dict[str, Any]:
@@ -87,7 +87,7 @@ def execute_tasks_v2(
         Any,
     ],
     job_type_filters: dict[str, bool],
-    job_attribute_filters: AttributeFiltersType,
+    job_attribute_filters: AttributeFilters,
 ) -> None:
     logger = logging.getLogger(logger_name)
 

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/runner/v2/task_interface.py

@@ -4,12 +4,11 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
-from pydantic import field_validator
 from pydantic import ValidationError
 
 from ....images import SingleImageTaskOutput
 from fractal_server.app.runner.exceptions import TaskOutputValidationError
-from fractal_server.urls import normalize_url
+from fractal_server.types import ZarrUrlStr
 
 
 class TaskOutput(BaseModel):
@@ -19,7 +18,7 @@ class TaskOutput(BaseModel):
     image_list_updates: list[SingleImageTaskOutput] = Field(
         default_factory=list
     )
-    image_list_removals: list[str] = Field(default_factory=list)
+    image_list_removals: list[ZarrUrlStr] = Field(default_factory=list)
 
     def check_zarr_urls_are_unique(self) -> None:
         zarr_urls = [img.zarr_url for img in self.image_list_updates]
@@ -40,24 +39,14 @@ class TaskOutput(BaseModel):
                 msg = f"{msg}\n{duplicate}"
             raise ValueError(msg)
 
-    @field_validator("image_list_removals")
-    @classmethod
-    def normalize_paths(cls, v: list[str]) -> list[str]:
-        return [normalize_url(zarr_url) for zarr_url in v]
-
 
 class InitArgsModel(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    zarr_url: str
+    zarr_url: ZarrUrlStr
     init_args: dict[str, Any] = Field(default_factory=dict)
 
-    @field_validator("zarr_url")
-    @classmethod
-    def normalize_path(cls, v: str) -> str:
-        return normalize_url(v)
-
 
 class InitTaskOutput(BaseModel):
 

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/schemas/user.py

@@ -3,12 +3,11 @@ from typing import Optional
 from fastapi_users import schemas
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import EmailStr
 from pydantic import Field
-from pydantic import field_validator
-from pydantic import ValidationInfo
 
-from ._validators import NonEmptyString
-from ._validators import val_unique_list
+from fractal_server.types import ListUniqueNonNegativeInt
+from fractal_server.types import NonEmptyStr
 
 __all__ = (
     "UserRead",
@@ -56,24 +55,12 @@ class UserUpdate(schemas.BaseUserUpdate):
     """
 
     model_config = ConfigDict(extra="forbid")
-
-    username: Optional[NonEmptyString] = None
-
-    # Validators
-
-    @field_validator(
-        "username",
-        "is_active",
-        "is_verified",
-        "is_superuser",
-        "email",
-        "password",
-    )
-    @classmethod
-    def cant_set_none(cls, v, info: ValidationInfo):
-        if v is None:
-            raise ValueError(f"Cannot set {info.field_name}=None")
-        return v
+    username: NonEmptyStr = None
+    password: NonEmptyStr = None
+    email: EmailStr = None
+    is_active: bool = None
+    is_superuser: bool = None
+    is_verified: bool = None
 
 
 class UserUpdateStrict(BaseModel):
@@ -94,14 +81,7 @@ class UserCreate(schemas.BaseUserCreate):
         username:
     """
 
-    username: Optional[NonEmptyString] = None
-
-    @field_validator("username")
-    @classmethod
-    def cant_set_none(cls, v, info: ValidationInfo):
-        if v is None:
-            raise ValueError(f"Cannot set {info.field_name}=None")
-        return v
+    username: NonEmptyStr = None
 
 
 class UserUpdateGroups(BaseModel):
@@ -112,8 +92,4 @@ class UserUpdateGroups(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    group_ids: list[int] = Field(min_length=1)
-
-    _group_ids = field_validator("group_ids")(
-        classmethod(val_unique_list("group_ids"))
-    )
+    group_ids: ListUniqueNonNegativeInt = Field(min_length=1)

{fractal_server-2.14.3 → fractal_server-2.14.4a0}/fractal_server/app/schemas/user_group.py

@@ -5,11 +5,9 @@ from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
 from pydantic import field_serializer
-from pydantic import field_validator
 from pydantic.types import AwareDatetime
 
-from ._validators import val_absolute_path
-from ._validators import val_unique_list
+from fractal_server.types import ListUniqueAbsolutePathStr
 
 __all__ = (
     "UserGroupRead",
@@ -54,15 +52,7 @@ class UserGroupCreate(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     name: str
-    viewer_paths: list[str] = Field(default_factory=list)
-
-    @field_validator("viewer_paths")
-    @classmethod
-    def viewer_paths_validator(cls, value):
-        for i, path in enumerate(value):
-            value[i] = val_absolute_path(f"viewer_paths[{i}]")(cls, path)
-        value = val_unique_list("viewer_paths")(cls, value)
-        return value
+    viewer_paths: ListUniqueAbsolutePathStr = Field(default_factory=list)
 
 
 class UserGroupUpdate(BaseModel):
@@ -72,14 +62,4 @@ class UserGroupUpdate(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    viewer_paths: Optional[list[str]] = None
-
-    @field_validator("viewer_paths")
-    @classmethod
-    def viewer_paths_validator(cls, value):
-        if value is None:
-            raise ValueError("Cannot set `viewer_paths=None`.")
-        for i, path in enumerate(value):
-            value[i] = val_absolute_path(f"viewer_paths[{i}]")(cls, path)
-        value = val_unique_list("viewer_paths")(cls, value)
-        return value
+    viewer_paths: ListUniqueAbsolutePathStr = None

fractal_server-2.14.4a0/fractal_server/app/schemas/user_settings.py

@@ -0,0 +1,69 @@
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import field_validator
+
+from fractal_server.string_tools import validate_cmd
+from fractal_server.types import AbsolutePathStr
+from fractal_server.types import ListUniqueNonEmptyString
+from fractal_server.types import NonEmptyStr
+
+__all__ = (
+    "UserSettingsRead",
+    "UserSettingsReadStrict",
+    "UserSettingsUpdate",
+    "UserSettingsUpdateStrict",
+)
+
+
+class UserSettingsRead(BaseModel):
+    """
+    Schema reserved for superusers
+    """
+
+    id: int
+    ssh_host: Optional[str] = None
+    ssh_username: Optional[str] = None
+    ssh_private_key_path: Optional[str] = None
+    ssh_tasks_dir: Optional[str] = None
+    ssh_jobs_dir: Optional[str] = None
+    slurm_user: Optional[str] = None
+    slurm_accounts: list[str]
+    project_dir: Optional[str] = None
+
+
+class UserSettingsReadStrict(BaseModel):
+    slurm_user: Optional[str] = None
+    slurm_accounts: list[str]
+    ssh_username: Optional[str] = None
+    project_dir: Optional[str] = None
+
+
+class UserSettingsUpdate(BaseModel):
+    """
+    Schema reserved for superusers
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    ssh_host: Optional[NonEmptyStr] = None
+    ssh_username: Optional[NonEmptyStr] = None
+    ssh_private_key_path: Optional[AbsolutePathStr] = None
+    ssh_tasks_dir: Optional[AbsolutePathStr] = None
+    ssh_jobs_dir: Optional[AbsolutePathStr] = None
+    slurm_user: Optional[NonEmptyStr] = None
+    slurm_accounts: Optional[ListUniqueNonEmptyString] = None
+    project_dir: Optional[AbsolutePathStr] = None
+
+    @field_validator("project_dir", mode="after")
+    @classmethod
+    def validate_project_dir(cls, value):
+        if value is not None:
+            validate_cmd(value)
+        return value
+
+
+class UserSettingsUpdateStrict(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    slurm_accounts: Optional[ListUniqueNonEmptyString] = None