fractal-server 2.10.5__py3-none-any.whl → 2.11.0a2__py3-none-any.whl

fractal_server/app/schemas/v2/workflowtask.py

@@ -6,14 +6,16 @@ from typing import Union
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
-from .._validators import valdictkeys
+from .._filter_validators import validate_type_filters
+from .._validators import root_validate_dict_keys
+from .._validators import valdict_keys
 from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
-from fractal_server.images import Filters
 
 RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
 
@@ -43,21 +45,28 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -71,7 +80,7 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -101,7 +110,7 @@ class WorkflowTaskReadV2(BaseModel):
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
 
-    input_filters: Filters
+    type_filters: dict[str, bool]
 
     task_type: str
     task_id: int
@@ -118,21 +127,28 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Optional[Filters]
+    type_filters: Optional[dict[str, bool]]
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -146,7 +162,7 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -164,21 +180,28 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
 
-    input_filters: Optional[Filters] = None
+    type_filters: Optional[dict[str, bool]] = None
 
     task: Union[TaskImportV2, TaskImportV2Legacy]
 
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
     _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
-        valdictkeys("args_non_parallel")
+        valdict_keys("args_non_parallel")
     )
     _args_parallel = validator("args_parallel", allow_reuse=True)(
-        valdictkeys("args_parallel")
+        valdict_keys("args_parallel")
     )
 
 
@@ -188,6 +211,6 @@ class WorkflowTaskExportV2(BaseModel):
     meta_parallel: Optional[dict[str, Any]] = None
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     task: TaskExportV2

fractal_server/data_migrations/2_11_0.py

@@ -0,0 +1,67 @@
+import logging
+
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import DatasetV2
+from fractal_server.app.models import JobV2
+from fractal_server.app.models import WorkflowTaskV2
+
+logger = logging.getLogger("fix_db")
+logger.setLevel(logging.INFO)
+
+
+def fix_db():
+
+    logger.info("START execution of fix_db function")
+
+    with next(get_sync_db()) as db:
+
+        # DatasetV2.filters
+        # DatasetV2.history[].workflowtask.input_filters
+        stm = select(DatasetV2).order_by(DatasetV2.id)
+        datasets = db.execute(stm).scalars().all()
+        for ds in datasets:
+            ds.attribute_filters = ds.filters["attributes"]
+            ds.type_filters = ds.filters["types"]
+            ds.filters = None
+            for i, h in enumerate(ds.history):
+                ds.history[i]["workflowtask"]["type_filters"] = h[
+                    "workflowtask"
+                ]["input_filters"]["types"]
+            flag_modified(ds, "history")
+            db.add(ds)
+            logger.info(f"Fixed filters in DatasetV2[{ds.id}]")
+
+        # WorkflowTaskV2.input_filters
+        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
+        wftasks = db.execute(stm).scalars().all()
+        for wft in wftasks:
+            wft.type_filters = wft.input_filters["types"]
+            if wft.input_filters["attributes"]:
+                logger.warning(
+                    f"Removing WorkflowTaskV2[{wft.id}].input_filters"
+                    f"['attributes'] = {wft.input_filters['attributes']}"
+                )
+            wft.input_filters = None
+            flag_modified(wft, "input_filters")
+            db.add(wft)
+            logger.info(f"Fixed filters in WorkflowTaskV2[{wft.id}]")
+
+        # JOBS V2
+        stm = select(JobV2).order_by(JobV2.id)
+        jobs = db.execute(stm).scalars().all()
+        for job in jobs:
+            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
+                "types"
+            ]
+            job.dataset_dump["attribute_filters"] = job.dataset_dump[
+                "filters"
+            ]["attributes"]
+            job.dataset_dump.pop("filters")
+            flag_modified(job, "dataset_dump")
+            logger.info(f"Fixed filters in JobV2[{job.id}].datasetdump")
+
+        db.commit()
+        logger.info("Changes committed.")

fractal_server/images/__init__.py

@@ -1,4 +1,3 @@
-from .models import Filters  # noqa: F401
 from .models import SingleImage  # noqa: F401
 from .models import SingleImageTaskOutput  # noqa: F401
 from .models import SingleImageUpdate  # noqa: F401

fractal_server/images/models.py

@@ -3,15 +3,16 @@ from typing import Optional
 from typing import Union
 
 from pydantic import BaseModel
-from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
-from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.app.schemas._validators import valdict_keys
 from fractal_server.urls import normalize_url
 
+AttributeFiltersType = dict[str, Optional[list[Any]]]
 
-class SingleImageBase(BaseModel):
+
+class _SingleImageBase(BaseModel):
     """
     Base for SingleImage and SingleImageTaskOutput.
 
@@ -30,9 +31,9 @@ class SingleImageBase(BaseModel):
 
     # Validators
     _attributes = validator("attributes", allow_reuse=True)(
-        valdictkeys("attributes")
+        valdict_keys("attributes")
     )
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
+    _types = validator("types", allow_reuse=True)(valdict_keys("types"))
 
     @validator("zarr_url")
     def normalize_zarr_url(cls, v: str) -> str:
@@ -44,7 +45,7 @@ class SingleImageBase(BaseModel):
             return normalize_url(v)
 
 
-class SingleImageTaskOutput(SingleImageBase):
+class SingleImageTaskOutput(_SingleImageBase):
     """
     `SingleImageBase`, with scalar `attributes` values (`None` included).
     """
@@ -63,7 +64,7 @@ class SingleImageTaskOutput(SingleImageBase):
         return v
 
 
-class SingleImage(SingleImageBase):
+class SingleImage(_SingleImageBase):
     """
     `SingleImageBase`, with scalar `attributes` values (`None` excluded).
     """
@@ -83,8 +84,8 @@ class SingleImage(SingleImageBase):
 
 class SingleImageUpdate(BaseModel):
     zarr_url: str
-    attributes: Optional[dict[str, Any]]
-    types: Optional[dict[str, bool]]
+    attributes: Optional[dict[str, Any]] = None
+    types: Optional[dict[str, bool]] = None
 
     @validator("zarr_url")
     def normalize_zarr_url(cls, v: str) -> str:
@@ -96,7 +97,7 @@ class SingleImageUpdate(BaseModel):
     ) -> dict[str, Union[int, float, str, bool]]:
         if v is not None:
             # validate keys
-            valdictkeys("attributes")(v)
+            valdict_keys("attributes")(v)
             # validate values
             for key, value in v.items():
                 if not isinstance(value, (int, float, str, bool)):
@@ -107,28 +108,4 @@ class SingleImageUpdate(BaseModel):
                     )
         return v
 
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
-
-
-class Filters(BaseModel, extra=Extra.forbid):
-    attributes: dict[str, Any] = Field(default_factory=dict)
-    types: dict[str, bool] = Field(default_factory=dict)
-
-    # Validators
-    _attributes = validator("attributes", allow_reuse=True)(
-        valdictkeys("attributes")
-    )
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
-
-    @validator("attributes")
-    def validate_attributes(
-        cls, v: dict[str, Any]
-    ) -> dict[str, Union[int, float, str, bool, None]]:
-        for key, value in v.items():
-            if not isinstance(value, (int, float, str, bool, type(None))):
-                raise ValueError(
-                    f"Filters.attributes[{key}] must be a scalar "
-                    "(int, float, str, bool, or None). "
-                    f"Given {value} ({type(value)})"
-                )
-        return v
+    _types = validator("types", allow_reuse=True)(valdict_keys("types"))

fractal_server/images/tools.py

@@ -4,8 +4,7 @@ from typing import Literal
 from typing import Optional
 from typing import Union
 
-from fractal_server.images import Filters
-
+from fractal_server.images.models import AttributeFiltersType
 
 ImageSearch = dict[Literal["image", "index"], Union[int, dict[str, Any]]]
 
@@ -33,52 +32,69 @@ def find_image_by_zarr_url(
     return dict(image=copy(images[ind]), index=ind)
 
 
-def match_filter(image: dict[str, Any], filters: Filters) -> bool:
+def match_filter(
+    *,
+    image: dict[str, Any],
+    type_filters: dict[str, bool],
+    attribute_filters: AttributeFiltersType,
+) -> bool:
     """
     Find whether an image matches a filter set.
 
     Arguments:
         image: A single image.
-        filters: A set of filters.
+        type_filters:
+        attribute_filters:
 
     Returns:
         Whether the image matches the filter set.
     """
+
     # Verify match with types (using a False default)
-    for key, value in filters.types.items():
+    for key, value in type_filters.items():
         if image["types"].get(key, False) != value:
             return False
-    # Verify match with attributes (only for non-None filters)
-    for key, value in filters.attributes.items():
-        if value is None:
+
+    # Verify match with attributes (only for not-None filters)
+    for key, values in attribute_filters.items():
+        if values is None:
             continue
-        if image["attributes"].get(key) != value:
+        if image["attributes"].get(key) not in values:
             return False
+
     return True
 
 
 def filter_image_list(
     images: list[dict[str, Any]],
-    filters: Filters,
+    type_filters: Optional[dict[str, bool]] = None,
+    attribute_filters: Optional[AttributeFiltersType] = None,
 ) -> list[dict[str, Any]]:
     """
     Compute a sublist with images that match a filter set.
 
     Arguments:
         images: A list of images.
-        filters: A set of filters.
+        type_filters:
+        attribute_filters:
 
     Returns:
         List of the `images` elements which match the filter set.
     """
 
     # When no filter is provided, return all images
-    if filters.attributes == {} and filters.types == {}:
+    if type_filters is None and attribute_filters is None:
         return images
+    actual_type_filters = type_filters or {}
+    actual_attribute_filters = attribute_filters or {}
 
     filtered_images = [
         copy(this_image)
         for this_image in images
-        if match_filter(this_image, filters=filters)
+        if match_filter(
+            image=this_image,
+            type_filters=actual_type_filters,
+            attribute_filters=actual_attribute_filters,
+        )
     ]
     return filtered_images

fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py

@@ -0,0 +1,96 @@
+"""split filters and keep old columns
+
+Revision ID: db09233ad13a
+Revises: 316140ff7ee1
+Create Date: 2025-01-14 14:50:46.007222
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "db09233ad13a"
+down_revision = "316140ff7ee1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                sa.JSON(),
+                server_default="{}",
+                nullable=False,
+            )
+        )
+        batch_op.alter_column(
+            "filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+            server_default="null",
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                sa.JSON(),
+                server_default="{}",
+                nullable=False,
+            )
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+        batch_op.alter_column(
+            "input_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+            server_default="null",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "input_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+            existing_server_default=sa.text(
+                '\'{"attributes": {}, "types": {}}\'::json'
+            ),
+        )
+        batch_op.drop_column("type_filters")
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_column("attribute_filters")
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+            existing_server_default=sa.text(
+                '\'{"attributes": {}, "types": {}}\'::json'
+            ),
+        )
+        batch_op.drop_column("attribute_filters")
+        batch_op.drop_column("type_filters")
+
+    # ### end Alembic commands ###

{fractal_server-2.10.5.dist-info → fractal_server-2.11.0a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.10.5
+Version: 2.11.0a2
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause