fractal-server 2.13.0__py3-none-any.whl → 2.14.0__py3-none-any.whl

fractal_server/app/schemas/user_settings.py

@@ -5,9 +5,9 @@ from pydantic import ConfigDict
 from pydantic import field_validator
 from pydantic.types import StrictStr
 
+from ._validators import NonEmptyString
 from ._validators import val_absolute_path
 from ._validators import val_unique_list
-from ._validators import valstr
 from fractal_server.string_tools import validate_cmd
 
 __all__ = (
@@ -48,21 +48,15 @@ class UserSettingsUpdate(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    ssh_host: Optional[str] = None
-    ssh_username: Optional[str] = None
-    ssh_private_key_path: Optional[str] = None
-    ssh_tasks_dir: Optional[str] = None
-    ssh_jobs_dir: Optional[str] = None
-    slurm_user: Optional[str] = None
-    slurm_accounts: Optional[list[StrictStr]] = None
-    project_dir: Optional[str] = None
+    ssh_host: Optional[NonEmptyString] = None
+    ssh_username: Optional[NonEmptyString] = None
+    ssh_private_key_path: Optional[NonEmptyString] = None
+    ssh_tasks_dir: Optional[NonEmptyString] = None
+    ssh_jobs_dir: Optional[NonEmptyString] = None
+    slurm_user: Optional[NonEmptyString] = None
+    slurm_accounts: Optional[list[NonEmptyString]] = None
+    project_dir: Optional[NonEmptyString] = None
 
-    _ssh_host = field_validator("ssh_host")(
-        classmethod(valstr("ssh_host", accept_none=True))
-    )
-    _ssh_username = field_validator("ssh_username")(
-        classmethod(valstr("ssh_username", accept_none=True))
-    )
     _ssh_private_key_path = field_validator("ssh_private_key_path")(
         classmethod(
             val_absolute_path("ssh_private_key_path", accept_none=True)
@@ -76,17 +70,11 @@ class UserSettingsUpdate(BaseModel):
         classmethod(val_absolute_path("ssh_jobs_dir", accept_none=True))
     )
 
-    _slurm_user = field_validator("slurm_user")(
-        classmethod(valstr("slurm_user", accept_none=True))
-    )
-
     @field_validator("slurm_accounts")
     @classmethod
     def slurm_accounts_validator(cls, value):
         if value is None:
             return value
-        for i, item in enumerate(value):
-            value[i] = valstr(f"slurm_accounts[{i}]")(cls, item)
         return val_unique_list("slurm_accounts")(cls, value)
 
     @field_validator("project_dir")

fractal_server/app/schemas/v2/__init__.py

@@ -1,3 +1,4 @@
+from .accounting import AccountingRecordRead  # noqa F401
 from .dataset import DatasetCreateV2  # noqa F401
 from .dataset import DatasetExportV2  # noqa F401
 from .dataset import DatasetImportV2  # noqa F401
@@ -6,8 +7,16 @@ from .dataset import DatasetUpdateV2  # noqa F401
 from .dumps import DatasetDumpV2  # noqa F401
 from .dumps import ProjectDumpV2  # noqa F401
 from .dumps import TaskDumpV2  # noqa F401
+from .dumps import TaskGroupDumpV2  # noqa F401
 from .dumps import WorkflowDumpV2  # noqa F401
 from .dumps import WorkflowTaskDumpV2  # noqa F401
+from .history import HistoryRunRead  # noqa F401
+from .history import HistoryRunReadAggregated  # noqa F401
+from .history import HistoryUnitRead  # noqa F401
+from .history import HistoryUnitStatus  # noqa F401
+from .history import HistoryUnitStatusQuery  # noqa F401
+from .history import ImageLogsRequest  # noqa F401
+from .history import SingleImageWithStatus  # noqa F401
 from .job import JobCreateV2  # noqa F401
 from .job import JobReadV2  # noqa F401
 from .job import JobStatusTypeV2  # noqa F401
@@ -17,6 +26,7 @@ from .manifest import TaskManifestV2  # noqa F401
 from .project import ProjectCreateV2  # noqa F401
 from .project import ProjectReadV2  # noqa F401
 from .project import ProjectUpdateV2  # noqa F401
+from .status_legacy import WorkflowTaskStatusTypeV2  # noqa F401
 from .task import TaskCreateV2  # noqa F401
 from .task import TaskExportV2  # noqa F401
 from .task import TaskImportV2  # noqa F401
@@ -46,5 +56,4 @@ from .workflowtask import WorkflowTaskImportV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2  # noqa F401
 from .workflowtask import WorkflowTaskReadV2WithWarning  # noqa F401
 from .workflowtask import WorkflowTaskReplaceV2  # noqa F401
-from .workflowtask import WorkflowTaskStatusTypeV2  # noqa F401
 from .workflowtask import WorkflowTaskUpdateV2  # noqa F401

fractal_server/app/schemas/v2/accounting.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from pydantic import field_serializer
+from pydantic.types import AwareDatetime
+
+
+class AccountingRecordRead(BaseModel):
+
+    id: int
+    user_id: int
+    timestamp: AwareDatetime
+    num_tasks: int
+    num_new_images: int
+
+    @field_serializer("timestamp")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()

fractal_server/app/schemas/v2/dataset.py

@@ -1,5 +1,4 @@
 from datetime import datetime
-from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
@@ -10,39 +9,22 @@ from pydantic import field_validator
 from pydantic import model_validator
 from pydantic.types import AwareDatetime
 
-from .._filter_validators import validate_attribute_filters
-from .._filter_validators import validate_type_filters
+from .._validators import cant_set_none
+from .._validators import NonEmptyString
 from .._validators import root_validate_dict_keys
-from .._validators import valstr
-from .dumps import WorkflowTaskDumpV2
 from .project import ProjectReadV2
-from .workflowtask import WorkflowTaskStatusTypeV2
 from fractal_server.images import SingleImage
 from fractal_server.images.models import AttributeFiltersType
 from fractal_server.urls import normalize_url
 
 
-class _DatasetHistoryItemV2(BaseModel):
-    """
-    Class for an item of `Dataset.history`.
-    """
-
-    workflowtask: WorkflowTaskDumpV2
-    status: WorkflowTaskStatusTypeV2
-    parallelization: Optional[dict] = None
-
-
-# CRUD
-
-
 class DatasetCreateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
-    name: str
+    name: NonEmptyString
 
     zarr_dir: Optional[str] = None
 
-    type_filters: dict[str, bool] = Field(default_factory=dict)
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
     # Validators
@@ -50,14 +32,6 @@ class DatasetCreateV2(BaseModel):
     _dict_keys = model_validator(mode="before")(
         classmethod(root_validate_dict_keys)
     )
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
-
-    _name = field_validator("name")(classmethod(valstr("name")))
 
     @field_validator("zarr_dir")
     @classmethod
@@ -74,13 +48,9 @@ class DatasetReadV2(BaseModel):
     project_id: int
     project: ProjectReadV2
 
-    history: list[_DatasetHistoryItemV2]
-
     timestamp_created: AwareDatetime
 
     zarr_dir: str
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType
 
     @field_serializer("timestamp_created")
     def serialize_datetime(v: datetime) -> str:
@@ -90,24 +60,19 @@ class DatasetReadV2(BaseModel):
 class DatasetUpdateV2(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
-    name: Optional[str] = None
+    name: Optional[NonEmptyString] = None
     zarr_dir: Optional[str] = None
-    type_filters: Optional[dict[str, bool]] = None
-    attribute_filters: Optional[dict[str, list[Any]]] = None
 
     # Validators
 
     _dict_keys = model_validator(mode="before")(
         classmethod(root_validate_dict_keys)
     )
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
 
-    _name = field_validator("name")(classmethod(valstr("name")))
+    @field_validator("name")
+    @classmethod
+    def _cant_set_none(cls, v):
+        return cant_set_none(v)
 
     @field_validator("zarr_dir")
     @classmethod
@@ -121,63 +86,20 @@ class DatasetImportV2(BaseModel):
     """
     Class for `Dataset` import.
 
+    We are dropping `model_config = ConfigDict(extra="forbid")` so that any
+    kind of legacy filters can be included in the payload, and ignored in the
+    API.
+
     Attributes:
         name:
         zarr_dir:
         images:
-        filters:
-        type_filters:
-        attribute_filters:
     """
 
-    model_config = ConfigDict(extra="forbid")
-
     name: str
     zarr_dir: str
     images: list[SingleImage] = Field(default_factory=list)
 
-    filters: Optional[dict[str, Any]] = None
-    type_filters: dict[str, bool] = Field(default_factory=dict)
-    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
-
-    @model_validator(mode="before")
-    @classmethod
-    def update_legacy_filters(cls, values: dict):
-        """
-        Transform legacy filters (created with fractal-server<2.11.0)
-        into attribute/type filters
-        """
-        if values.get("filters") is not None:
-            if (
-                "type_filters" in values.keys()
-                or "attribute_filters" in values.keys()
-            ):
-                raise ValueError(
-                    "Cannot set filters both through the legacy field "
-                    "('filters') and the new ones ('type_filters' and/or "
-                    "'attribute_filters')."
-                )
-
-            else:
-                # Convert legacy filters.types into new type_filters
-                values["type_filters"] = values["filters"].get("types", {})
-                values["attribute_filters"] = {
-                    key: [value]
-                    for key, value in values["filters"]
-                    .get("attributes", {})
-                    .items()
-                }
-                values["filters"] = None
-
-        return values
-
-    _type_filters = field_validator("type_filters")(
-        classmethod(validate_type_filters)
-    )
-    _attribute_filters = field_validator("attribute_filters")(
-        classmethod(validate_attribute_filters)
-    )
-
     @field_validator("zarr_dir")
     @classmethod
     def normalize_zarr_dir(cls, v: str) -> str:
@@ -192,12 +114,8 @@ class DatasetExportV2(BaseModel):
         name:
         zarr_dir:
         images:
-        type_filters:
-        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage]
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/dumps.py

@@ -1,19 +1,20 @@
 """
-
 Dump models differ from their Read counterpart in that:
 * They are directly JSON-able, without any additional encoder.
-* They may only include a subset of the Read attributes.
+* They may include only a subset of the available fields.
 
 These models are used in at least two situations:
 1. In the "*_dump" attributes of Job models;
-2. In the `_DatasetHistoryItem.workflowtask` model, to trim its size.
+2. In the history items, to trim their size.
 """
 from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import ConfigDict
+from pydantic import Field
 
-from fractal_server.images.models import AttributeFiltersType
+from .task import TaskTypeType
+from .task_group import TaskGroupV2OriginEnum
 
 
 class ProjectDumpV2(BaseModel):
@@ -26,7 +27,7 @@ class ProjectDumpV2(BaseModel):
 class TaskDumpV2(BaseModel):
     id: int
     name: str
-    type: str
+    type: TaskTypeType
 
     command_non_parallel: Optional[str] = None
     command_parallel: Optional[str] = None
@@ -63,12 +64,28 @@ class WorkflowDumpV2(BaseModel):
 
 
 class DatasetDumpV2(BaseModel):
-    model_config = ConfigDict(extra="forbid")
+    """
+    We do not include 'model_config = ConfigDict(extra="forbid")' because
+    legacy data may include 'type_filters' or 'attribute_filters' and we
+    want to avoid response-validation errors.
+    """
+
     id: int
     name: str
     project_id: int
     timestamp_created: str
-
     zarr_dir: str
-    type_filters: dict[str, bool]
-    attribute_filters: AttributeFiltersType
+
+
+class TaskGroupDumpV2(BaseModel):
+    id: int
+    origin: TaskGroupV2OriginEnum
+    pkg_name: str
+    version: Optional[str] = None
+    python_version: Optional[str] = None
+    pip_extras: Optional[str] = None
+    pinned_package_versions: dict[str, str] = Field(default_factory=dict)
+
+    path: Optional[str] = None
+    venv_path: Optional[str] = None
+    wheel_path: Optional[str] = None

fractal_server/app/schemas/v2/history.py

@@ -0,0 +1,80 @@
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from typing import Optional
+
+from pydantic import AwareDatetime
+from pydantic import BaseModel
+from pydantic import field_serializer
+
+from ....images import SingleImage
+
+
+class HistoryUnitStatus(str, Enum):
+    """
+    Available status for images
+
+    Attributes:
+        SUBMITTED:
+        DONE:
+        FAILED:
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class HistoryUnitStatusQuery(str, Enum):
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+    UNSET = "unset"
+
+
+class HistoryUnitRead(BaseModel):
+    id: int
+    logfile: Optional[str] = None
+    status: HistoryUnitStatus
+    zarr_urls: list[str]
+
+
+class HistoryRunRead(BaseModel):
+    id: int
+    dataset_id: int
+    workflowtask_id: Optional[int] = None
+    job_id: int
+    workflowtask_dump: dict[str, Any]
+    task_group_dump: dict[str, Any]
+    timestamp_started: AwareDatetime
+    status: HistoryUnitStatus
+    num_available_images: int
+
+    @field_serializer("timestamp_started")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()
+
+
+class HistoryRunReadAggregated(BaseModel):
+    id: int
+    timestamp_started: AwareDatetime
+    workflowtask_dump: dict[str, Any]
+    num_submitted_units: int
+    num_done_units: int
+    num_failed_units: int
+
+    @field_serializer("timestamp_started")
+    def serialize_datetime(v: datetime) -> str:
+        return v.isoformat()
+
+
+class ImageLogsRequest(BaseModel):
+    workflowtask_id: int
+    dataset_id: int
+    zarr_url: str
+
+
+class SingleImageWithStatus(SingleImage):
+    status: Optional[HistoryUnitStatus] = None

fractal_server/app/schemas/v2/job.py

@@ -13,8 +13,10 @@ from pydantic.types import AwareDatetime
 from pydantic.types import StrictStr
 
 from .._filter_validators import validate_attribute_filters
+from .._filter_validators import validate_type_filters
+from .._validators import cant_set_none
+from .._validators import NonEmptyString
 from .._validators import root_validate_dict_keys
-from .._validators import valstr
 from .dumps import DatasetDumpV2
 from .dumps import ProjectDumpV2
 from .dumps import WorkflowDumpV2
@@ -43,26 +45,32 @@ class JobStatusTypeV2(str, Enum):
 
 
 class JobCreateV2(BaseModel):
-
     model_config = ConfigDict(extra="forbid")
 
     first_task_index: Optional[int] = None
     last_task_index: Optional[int] = None
     slurm_account: Optional[StrictStr] = None
-    worker_init: Optional[str] = None
+    worker_init: Optional[NonEmptyString] = None
 
     attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     # Validators
-    _worker_init = field_validator("worker_init")(
-        classmethod(valstr("worker_init"))
-    )
+
+    @field_validator("worker_init")
+    @classmethod
+    def _cant_set_none(cls, v):
+        return cant_set_none(v)
+
     _dict_keys = model_validator(mode="before")(
         classmethod(root_validate_dict_keys)
     )
     _attribute_filters = field_validator("attribute_filters")(
         classmethod(validate_attribute_filters)
     )
+    _type_filters = field_validator("type_filters")(
+        classmethod(validate_type_filters)
+    )
 
     @field_validator("first_task_index")
     @classmethod
@@ -100,7 +108,6 @@ class JobCreateV2(BaseModel):
 
 
 class JobReadV2(BaseModel):
-
     id: int
     project_id: Optional[int] = None
     project_dump: ProjectDumpV2
@@ -120,6 +127,7 @@ class JobReadV2(BaseModel):
     last_task_index: Optional[int] = None
     worker_init: Optional[str] = None
     attribute_filters: AttributeFiltersType
+    type_filters: dict[str, bool]
 
     @field_serializer("start_timestamp")
     def serialize_datetime_start(v: datetime) -> str:
@@ -134,7 +142,6 @@ class JobReadV2(BaseModel):
 
 
 class JobUpdateV2(BaseModel):
-
     model_config = ConfigDict(extra="forbid")
 
     status: JobStatusTypeV2

fractal_server/app/schemas/v2/manifest.py

@@ -1,4 +1,5 @@
 from typing import Any
+from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
@@ -7,7 +8,7 @@ from pydantic import field_validator
 from pydantic import HttpUrl
 from pydantic import model_validator
 
-from .._validators import valstr
+from .._validators import NonEmptyString
 
 
 class TaskManifestV2(BaseModel):
@@ -56,6 +57,16 @@ class TaskManifestV2(BaseModel):
     modality: Optional[str] = None
     tags: list[str] = Field(default_factory=list)
 
+    type: Optional[
+        Literal[
+            "compound",
+            "converter_compound",
+            "non_parallel",
+            "converter_non_parallel",
+            "parallel",
+        ]
+    ] = None
+
     @model_validator(mode="after")
     def validate_executable_args_meta(self):
         executable_non_parallel = self.executable_non_parallel
@@ -128,7 +139,7 @@ class ManifestV2(BaseModel):
             The list of tasks, represented as specified by subclasses of the
             _TaskManifestBase (a.k.a. TaskManifestType)
         has_args_schemas:
-            `True` if the manifest incldues JSON Schemas for the arguments of
+            `True` if the manifest includes JSON Schemas for the arguments of
             each task.
         args_schema_version:
             Label of how `args_schema`s were generated (e.g. `pydantic_v1`).
@@ -138,7 +149,7 @@ class ManifestV2(BaseModel):
     task_list: list[TaskManifestV2]
     has_args_schemas: bool = False
     args_schema_version: Optional[str] = None
-    authors: Optional[str] = None
+    authors: Optional[NonEmptyString] = None
 
     @model_validator(mode="after")
     def _check_args_schemas_are_present(self):
@@ -181,7 +192,3 @@ class ManifestV2(BaseModel):
         if value != "2":
             raise ValueError(f"Wrong manifest version (given {value})")
         return value
-
-    _authors = field_validator("authors")(
-        classmethod(valstr("authors", accept_none=True))
-    )

fractal_server/app/schemas/v2/project.py

@@ -7,16 +7,15 @@ from pydantic import field_serializer
 from pydantic import field_validator
 from pydantic.types import AwareDatetime
 
-from .._validators import valstr
+from .._validators import cant_set_none
+from .._validators import NonEmptyString
 
 
 class ProjectCreateV2(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    name: str
-    # Validators
-    _name = field_validator("name")(classmethod(valstr("name")))
+    name: NonEmptyString
 
 
 class ProjectReadV2(BaseModel):
@@ -34,6 +33,9 @@ class ProjectUpdateV2(BaseModel):
 
     model_config = ConfigDict(extra="forbid")
 
-    name: Optional[str] = None
-    # Validators
-    _name = field_validator("name")(classmethod(valstr("name")))
+    name: Optional[NonEmptyString] = None
+
+    @field_validator("name")
+    @classmethod
+    def _cant_set_none(cls, v):
+        return cant_set_none(v)

fractal_server/app/schemas/v2/status_legacy.py

@@ -0,0 +1,35 @@
+from enum import Enum
+
+from pydantic import BaseModel
+from pydantic import Field
+
+
+class WorkflowTaskStatusTypeV2(str, Enum):
+    """
+    Define the available values for the status of a `WorkflowTask`.
+
+    This model is used within the `Dataset.history` attribute, which is
+    constructed in the runner and then used in the API (e.g. in the
+    `api/v2/project/{project_id}/dataset/{dataset_id}/status` endpoint).
+
+    Attributes:
+        SUBMITTED: The `WorkflowTask` is part of a running job.
+        DONE: The most-recent execution of this `WorkflowTask` was successful.
+        FAILED: The most-recent execution of this `WorkflowTask` failed.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class LegacyStatusReadV2(BaseModel):
+    """
+    Response type for the
+    `/project/{project_id}/status/` endpoint
+    """
+
+    status: dict[
+        str,
+        WorkflowTaskStatusTypeV2,
+    ] = Field(default_factory=dict)