fractal-server 2.11.0a0__py3-none-any.whl → 2.11.0a3__py3-none-any.whl

fractal_server/app/schemas/_filter_validators.py

@@ -0,0 +1,47 @@
+from typing import Optional
+
+from ._validators import valdict_keys
+from fractal_server.images.models import AttributeFiltersType
+
+
+def validate_type_filters(
+    type_filters: Optional[dict[str, bool]]
+) -> dict[str, bool]:
+    if type_filters is None:
+        raise ValueError("'type_filters' cannot be 'None'.")
+
+    type_filters = valdict_keys("type_filters")(type_filters)
+    return type_filters
+
+
+def validate_attribute_filters(
+    attribute_filters: Optional[AttributeFiltersType],
+) -> AttributeFiltersType:
+    if attribute_filters is None:
+        raise ValueError("'attribute_filters' cannot be 'None'.")
+
+    attribute_filters = valdict_keys("attribute_filters")(attribute_filters)
+    for key, values in attribute_filters.items():
+        if values is None:
+            # values=None corresponds to not applying any filter for
+            # attribute `key`
+            pass
+        elif values == []:
+            # WARNING: in this case, no image can match with the current
+            # filter. In the future we may deprecate this possibility.
+            pass
+        else:
+            # values is a non-empty list, and its items must all be of the
+            # same scalar non-None type
+            _type = type(values[0])
+            if not all(isinstance(value, _type) for value in values):
+                raise ValueError(
+                    f"attribute_filters[{key}] has values with "
+                    f"non-homogeneous types: {values}."
+                )
+            if _type not in (int, float, str, bool):
+                raise ValueError(
+                    f"attribute_filters[{key}] has values with "
+                    f"invalid types: {values}."
+                )
+    return attribute_filters

fractal_server/app/schemas/_validators.py

@@ -27,7 +27,7 @@ def valstr(attribute: str, accept_none: bool = False):
     return val
 
 
-def valdictkeys(attribute: str):
+def valdict_keys(attribute: str):
     def val(d: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
         """
         Apply valstr to every key of the dictionary, and fail if there are
@@ -38,7 +38,7 @@ def valdictkeys(attribute: str):
             new_keys = [valstr(f"{attribute}[{key}]")(key) for key in old_keys]
             if len(new_keys) != len(set(new_keys)):
                 raise ValueError(
-                    f"Dictionary contains multiple identical keys: {d}."
+                    f"Dictionary contains multiple identical keys: '{d}'."
                 )
             for old_key, new_key in zip(old_keys, new_keys):
                 if new_key != old_key:
@@ -101,3 +101,14 @@ def val_unique_list(attribute: str):
         return must_be_unique
 
     return val
+
+
+def root_validate_dict_keys(cls, object: dict) -> dict:
+    """
+    For each dictionary in `object.values()`,
+    checks that that dictionary has only keys of type str.
+    """
+    for dictionary in (v for v in object.values() if isinstance(v, dict)):
+        if not all(isinstance(key, str) for key in dictionary.keys()):
+            raise ValueError("Dictionary keys must be strings.")
+    return object

fractal_server/app/schemas/v2/dataset.py

@@ -1,17 +1,22 @@
 from datetime import datetime
+from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
+from .._filter_validators import validate_attribute_filters
+from .._filter_validators import validate_type_filters
+from .._validators import root_validate_dict_keys
 from .._validators import valstr
 from .dumps import WorkflowTaskDumpV2
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskStatusTypeV2
-from fractal_server.images import Filters
 from fractal_server.images import SingleImage
+from fractal_server.images.models import AttributeFiltersType
 from fractal_server.urls import normalize_url
 
 
@@ -34,17 +39,29 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
 
     zarr_dir: Optional[str] = None
 
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
     # Validators
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             return normalize_url(v)
         return v
 
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-
 
 class DatasetReadV2(BaseModel):
 
@@ -59,24 +76,37 @@ class DatasetReadV2(BaseModel):
     timestamp_created: datetime
 
     zarr_dir: str
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType
 
 
 class DatasetUpdateV2(BaseModel, extra=Extra.forbid):
 
     name: Optional[str]
     zarr_dir: Optional[str]
-    filters: Optional[Filters]
+    type_filters: Optional[dict[str, bool]]
+    attribute_filters: Optional[dict[str, list[Any]]]
 
     # Validators
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             return normalize_url(v)
         return v
 
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-
 
 class DatasetImportV2(BaseModel, extra=Extra.forbid):
     """
@@ -87,14 +117,55 @@ class DatasetImportV2(BaseModel, extra=Extra.forbid):
         zarr_dir:
         images:
         filters:
+        type_filters:
+        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage] = Field(default_factory=list)
-    filters: Filters = Field(default_factory=Filters)
 
-    # Validators
+    filters: Optional[dict[str, Any]] = None
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+
+    @root_validator(pre=True)
+    def update_legacy_filters(cls, values: dict):
+        """
+        Transform legacy filters (created with fractal-server<2.11.0)
+        into attribute/type filters
+        """
+        if values.get("filters") is not None:
+            if (
+                "type_filters" in values.keys()
+                or "attribute_filters" in values.keys()
+            ):
+                raise ValueError(
+                    "Cannot set filters both through the legacy field "
+                    "('filters') and the new ones ('type_filters' and/or "
+                    "'attribute_filters')."
+                )
+
+            else:
+                # Convert legacy filters.types into new type_filters
+                values["type_filters"] = values["filters"].get("types", {})
+                values["attribute_filters"] = {
+                    key: [value]
+                    for key, value in values["filters"]
+                    .get("attributes", {})
+                    .items()
+                }
+                values["filters"] = None
+
+        return values
+
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: str) -> str:
         return normalize_url(v)
@@ -108,10 +179,12 @@ class DatasetExportV2(BaseModel):
         name:
         zarr_dir:
         images:
-        filters:
+        type_filters:
+        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage]
-    filters: Filters
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/dumps.py

@@ -13,7 +13,7 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import Extra
 
-from fractal_server.images import Filters
+from fractal_server.images.models import AttributeFiltersType
 
 
 class ProjectDumpV2(BaseModel, extra=Extra.forbid):
@@ -39,19 +39,16 @@ class TaskDumpV2(BaseModel):
 
 class WorkflowTaskDumpV2(BaseModel):
     """
-    Before v2.5.0, WorkflowTaskV2 could have `task_id=task=None` and
-    non-`None` `task_legacy_id` and `task_legacy`. Since these objects
-    may still exist in the database after version updates, we are setting
-    `task_id` and `task` to `Optional` to avoid response-validation errors
+    We do not include 'extra=Extra.forbid' because legacy data may include
+    'input_filters' field and we want to avoid response-validation errors
     for the endpoints that GET datasets.
-    Ref issue #1783.
     """
 
     id: int
     workflow_id: int
     order: Optional[int]
 
-    input_filters: Filters
+    type_filters: dict[str, bool]
 
     task_id: Optional[int]
     task: Optional[TaskDumpV2]
@@ -71,4 +68,5 @@ class DatasetDumpV2(BaseModel, extra=Extra.forbid):
     timestamp_created: str
 
     zarr_dir: str
-    filters: Filters
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/job.py

@@ -4,13 +4,18 @@ from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
+from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 from pydantic.types import StrictStr
 
+from .._filter_validators import validate_attribute_filters
+from .._validators import root_validate_dict_keys
 from .._validators import valstr
 from .dumps import DatasetDumpV2
 from .dumps import ProjectDumpV2
 from .dumps import WorkflowDumpV2
+from fractal_server.images.models import AttributeFiltersType
 
 
 class JobStatusTypeV2(str, Enum):
@@ -41,10 +46,18 @@ class JobCreateV2(BaseModel, extra=Extra.forbid):
     slurm_account: Optional[StrictStr] = None
     worker_init: Optional[str]
 
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+
     # Validators
     _worker_init = validator("worker_init", allow_reuse=True)(
         valstr("worker_init")
     )
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
 
     @validator("first_task_index", always=True)
     def first_task_index_non_negative(cls, v, values):
@@ -99,6 +112,7 @@ class JobReadV2(BaseModel):
     first_task_index: Optional[int]
     last_task_index: Optional[int]
     worker_init: Optional[str]
+    attribute_filters: AttributeFiltersType
 
 
 class JobUpdateV2(BaseModel, extra=Extra.forbid):

fractal_server/app/schemas/v2/task.py

@@ -10,7 +10,7 @@ from pydantic import root_validator
 from pydantic import validator
 
 from fractal_server.app.schemas._validators import val_unique_list
-from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.app.schemas._validators import valdict_keys
 from fractal_server.app.schemas._validators import valstr
 from fractal_server.string_tools import validate_cmd
 
@@ -66,25 +66,25 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
     _version = validator("version", allow_reuse=True)(valstr("version"))
 
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
     _args_schema_non_parallel = validator(
         "args_schema_non_parallel", allow_reuse=True
-    )(valdictkeys("args_schema_non_parallel"))
+    )(valdict_keys("args_schema_non_parallel"))
     _args_schema_parallel = validator(
         "args_schema_parallel", allow_reuse=True
-    )(valdictkeys("args_schema_parallel"))
+    )(valdict_keys("args_schema_parallel"))
     _args_schema_version = validator("args_schema_version", allow_reuse=True)(
         valstr("args_schema_version")
     )
     _input_types = validator("input_types", allow_reuse=True)(
-        valdictkeys("input_types")
+        valdict_keys("input_types")
     )
     _output_types = validator("output_types", allow_reuse=True)(
-        valdictkeys("output_types")
+        valdict_keys("output_types")
     )
 
     _category = validator("category", allow_reuse=True)(
@@ -158,10 +158,10 @@ class TaskUpdateV2(BaseModel, extra=Extra.forbid):
         "command_non_parallel", allow_reuse=True
     )(valstr("command_non_parallel"))
     _input_types = validator("input_types", allow_reuse=True)(
-        valdictkeys("input_types")
+        valdict_keys("input_types")
     )
     _output_types = validator("output_types", allow_reuse=True)(
-        valdictkeys("output_types")
+        valdict_keys("output_types")
     )
 
     _category = validator("category", allow_reuse=True)(

fractal_server/app/schemas/v2/task_group.py

@@ -8,7 +8,7 @@ from pydantic import Field
 from pydantic import validator
 
 from .._validators import val_absolute_path
-from .._validators import valdictkeys
+from .._validators import valdict_keys
 from .._validators import valstr
 from .task import TaskReadV2
 
@@ -57,7 +57,7 @@ class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
     )
     _pinned_package_versions = validator(
         "pinned_package_versions", allow_reuse=True
-    )(valdictkeys("pinned_package_versions"))
+    )(valdict_keys("pinned_package_versions"))
     _pip_extras = validator("pip_extras", allow_reuse=True)(
         valstr("pip_extras")
     )

fractal_server/app/schemas/v2/workflowtask.py

@@ -6,14 +6,16 @@ from typing import Union
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
-from .._validators import valdictkeys
+from .._filter_validators import validate_type_filters
+from .._validators import root_validate_dict_keys
+from .._validators import valdict_keys
 from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
-from fractal_server.images import Filters
 
 RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
 
@@ -43,21 +45,28 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -71,7 +80,7 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -101,7 +110,7 @@ class WorkflowTaskReadV2(BaseModel):
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
 
-    input_filters: Filters
+    type_filters: dict[str, bool]
 
     task_type: str
     task_id: int
@@ -118,21 +127,28 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Optional[Filters]
+    type_filters: Optional[dict[str, bool]]
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -146,7 +162,7 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -163,22 +179,53 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]] = None
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
-
-    input_filters: Optional[Filters] = None
+    type_filters: Optional[dict[str, bool]] = None
+    filters: Optional[dict[str, Any]] = None
 
     task: Union[TaskImportV2, TaskImportV2Legacy]
 
+    # Validators
+    @root_validator(pre=True)
+    def update_legacy_filters(cls, values: dict):
+        """
+        Transform legacy filters (created with fractal-server<2.11.0)
+        into type filters
+        """
+        if values.get("filters") is not None:
+            if "type_filters" in values.keys():
+                raise ValueError(
+                    "Cannot set filters both through the legacy field "
+                    "('filters') and the new one ('type_filters')."
+                )
+
+            else:
+                # As of 2.11.0, WorkflowTask do not have attribute filters
+                # any more.
+                if values["filters"]["attributes"] != {}:
+                    raise ValueError(
+                        "Cannot set attribute filters for WorkflowTasks."
+                    )
+                # Convert legacy filters.types into new type_filters
+                values["type_filters"] = values["filters"].get("types", {})
+                values["filters"] = None
+
+        return values
+
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
     _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
-        valdictkeys("args_non_parallel")
+        valdict_keys("args_non_parallel")
     )
     _args_parallel = validator("args_parallel", allow_reuse=True)(
-        valdictkeys("args_parallel")
+        valdict_keys("args_parallel")
     )
 
 
@@ -188,6 +235,6 @@ class WorkflowTaskExportV2(BaseModel):
     meta_parallel: Optional[dict[str, Any]] = None
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     task: TaskExportV2

fractal_server/data_migrations/2_11_0.py

@@ -0,0 +1,67 @@
+import logging
+
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import DatasetV2
+from fractal_server.app.models import JobV2
+from fractal_server.app.models import WorkflowTaskV2
+
+logger = logging.getLogger("fix_db")
+logger.setLevel(logging.INFO)
+
+
+def fix_db():
+
+    logger.info("START execution of fix_db function")
+
+    with next(get_sync_db()) as db:
+
+        # DatasetV2.filters
+        # DatasetV2.history[].workflowtask.input_filters
+        stm = select(DatasetV2).order_by(DatasetV2.id)
+        datasets = db.execute(stm).scalars().all()
+        for ds in datasets:
+            ds.attribute_filters = ds.filters["attributes"]
+            ds.type_filters = ds.filters["types"]
+            ds.filters = None
+            for i, h in enumerate(ds.history):
+                ds.history[i]["workflowtask"]["type_filters"] = h[
+                    "workflowtask"
+                ]["input_filters"]["types"]
+            flag_modified(ds, "history")
+            db.add(ds)
+            logger.info(f"Fixed filters in DatasetV2[{ds.id}]")
+
+        # WorkflowTaskV2.input_filters
+        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
+        wftasks = db.execute(stm).scalars().all()
+        for wft in wftasks:
+            wft.type_filters = wft.input_filters["types"]
+            if wft.input_filters["attributes"]:
+                logger.warning(
+                    f"Removing WorkflowTaskV2[{wft.id}].input_filters"
+                    f"['attributes'] = {wft.input_filters['attributes']}"
+                )
+            wft.input_filters = None
+            flag_modified(wft, "input_filters")
+            db.add(wft)
+            logger.info(f"Fixed filters in WorkflowTaskV2[{wft.id}]")
+
+        # JOBS V2
+        stm = select(JobV2).order_by(JobV2.id)
+        jobs = db.execute(stm).scalars().all()
+        for job in jobs:
+            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
+                "types"
+            ]
+            job.dataset_dump["attribute_filters"] = job.dataset_dump[
+                "filters"
+            ]["attributes"]
+            job.dataset_dump.pop("filters")
+            flag_modified(job, "dataset_dump")
+            logger.info(f"Fixed filters in JobV2[{job.id}].datasetdump")
+
+        db.commit()
+        logger.info("Changes committed.")

fractal_server/images/__init__.py

@@ -1,4 +1,3 @@
-from .models import Filters  # noqa: F401
 from .models import SingleImage  # noqa: F401
 from .models import SingleImageTaskOutput  # noqa: F401
 from .models import SingleImageUpdate  # noqa: F401