fractal-server 2.10.6__py3-none-any.whl → 2.11.0__py3-none-any.whl

fractal_server/app/schemas/v2/workflowtask.py

@@ -6,14 +6,16 @@ from typing import Union
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
-from .._validators import valdictkeys
+from .._filter_validators import validate_type_filters
+from .._validators import root_validate_dict_keys
+from .._validators import valdict_keys
 from .task import TaskExportV2
 from .task import TaskImportV2
 from .task import TaskImportV2Legacy
 from .task import TaskReadV2
-from fractal_server.images import Filters
 
 RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
 
@@ -43,21 +45,28 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -71,7 +80,7 @@ class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -101,7 +110,7 @@ class WorkflowTaskReadV2(BaseModel):
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
 
-    input_filters: Filters
+    type_filters: dict[str, bool]
 
     task_type: str
     task_id: int
@@ -118,21 +127,28 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]]
     args_non_parallel: Optional[dict[str, Any]]
     args_parallel: Optional[dict[str, Any]]
-    input_filters: Optional[Filters]
+    type_filters: Optional[dict[str, bool]]
 
     # Validators
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
 
     @validator("args_non_parallel")
     def validate_args_non_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_non_parallel")(value)
+        valdict_keys("args_non_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -146,7 +162,7 @@ class WorkflowTaskUpdateV2(BaseModel, extra=Extra.forbid):
     def validate_args_parallel(cls, value):
         if value is None:
             return
-        valdictkeys("args_parallel")(value)
+        valdict_keys("args_parallel")(value)
         args_keys = set(value.keys())
         intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
         if intersect_keys:
@@ -163,22 +179,55 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
     meta_parallel: Optional[dict[str, Any]] = None
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
-
-    input_filters: Optional[Filters] = None
+    type_filters: Optional[dict[str, bool]] = None
+    input_filters: Optional[dict[str, Any]] = None
 
     task: Union[TaskImportV2, TaskImportV2Legacy]
 
+    # Validators
+    @root_validator(pre=True)
+    def update_legacy_filters(cls, values: dict):
+        """
+        Transform legacy filters (created with fractal-server<2.11.0)
+        into type filters
+        """
+        if values.get("input_filters") is not None:
+            if "type_filters" in values.keys():
+                raise ValueError(
+                    "Cannot set filters both through the legacy field "
+                    "('filters') and the new one ('type_filters')."
+                )
+
+            else:
+                # As of 2.11.0, WorkflowTask do not have attribute filters
+                # any more.
+                if values["input_filters"]["attributes"] != {}:
+                    raise ValueError(
+                        "Cannot set attribute filters for WorkflowTasks."
+                    )
+                # Convert legacy filters.types into new type_filters
+                values["type_filters"] = values["input_filters"].get(
+                    "types", {}
+                )
+                values["input_filters"] = None
+
+        return values
+
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
     _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
-        valdictkeys("args_non_parallel")
+        valdict_keys("args_non_parallel")
     )
     _args_parallel = validator("args_parallel", allow_reuse=True)(
-        valdictkeys("args_parallel")
+        valdict_keys("args_parallel")
     )
 
 
@@ -188,6 +237,6 @@ class WorkflowTaskExportV2(BaseModel):
     meta_parallel: Optional[dict[str, Any]] = None
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
-    input_filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
 
     task: TaskExportV2

fractal_server/data_migrations/2_11_0.py

@@ -0,0 +1,168 @@
+import logging
+from typing import Union
+
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import DatasetV2
+from fractal_server.app.models import JobV2
+from fractal_server.app.models import ProjectV2
+from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models import WorkflowV2
+from fractal_server.app.schemas.v2 import DatasetReadV2
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import ProjectReadV2
+from fractal_server.app.schemas.v2 import TaskReadV2
+from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
+from fractal_server.images.models import AttributeFiltersType
+
+logger = logging.getLogger("fix_db")
+logger.setLevel(logging.INFO)
+
+
+def dict_values_to_list(
+    input_dict: dict[str, Union[int, float, bool, str, None]],
+    identifier: str,
+) -> tuple[AttributeFiltersType, bool]:
+    was_there_a_warning = False
+    for k, v in input_dict.items():
+        if not isinstance(v, (int, float, bool, str, type(None))):
+            error_msg = (
+                f"Attribute '{k}' from '{identifier}' "
+                "has invalid type '{type(v)}'."
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
+        elif v is None:
+            logger.warning(
+                f"Attribute '{k}' from '{identifier}' is "
+                "None and it will be removed."
+            )
+            was_there_a_warning = True
+        else:
+            input_dict[k] = [v]
+    return input_dict, was_there_a_warning
+
+
+def fix_db():
+    logger.info("START execution of fix_db function")
+
+    with next(get_sync_db()) as db:
+        # DatasetV2.filters
+        stm = select(DatasetV2).order_by(DatasetV2.id)
+        datasets = db.execute(stm).scalars().all()
+        for ds in datasets:
+            logger.info(f"DatasetV2[{ds.id}] START")
+            if ds.filters is None:
+                logger.info(f"DatasetV2[{ds.id}] SKIP")
+                continue
+
+            ds.attribute_filters, warning = dict_values_to_list(
+                ds.filters["attributes"],
+                f"Dataset[{ds.id}].filters.attributes",
+            )
+            if warning:
+                proj = db.get(ProjectV2, ds.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{ds.id=}, "
+                    f"{ds.name=}"
+                )
+            ds.type_filters = ds.filters["types"]
+            ds.filters = None
+            for i, h in enumerate(ds.history):
+                ds.history[i]["workflowtask"]["type_filters"] = h[
+                    "workflowtask"
+                ]["input_filters"]["types"]
+                ds.history[i]["workflowtask"].pop("input_filters")
+            flag_modified(ds, "history")
+            DatasetReadV2(
+                **ds.model_dump(),
+                project=ProjectReadV2(**ds.project.model_dump()),
+            )
+            db.add(ds)
+            logger.info(f"DatasetV2[{ds.id}] END - fixed filters")
+
+        logger.info("------ switch from dataset to workflowtasks ------")
+
+        # WorkflowTaskV2.input_filters
+        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
+        wftasks = db.execute(stm).scalars().all()
+        for wft in wftasks:
+            logger.info(f"WorkflowTaskV2[{wft.id}] START")
+            if wft.input_filters is None:
+                logger.info(f"WorkflowTaskV2[{wft.id}] SKIP")
+                continue
+            wft.type_filters = wft.input_filters["types"]
+            if wft.input_filters["attributes"]:
+                logger.warning(
+                    "Removing input_filters['attributes']. "
+                    f"(previous value: {wft.input_filters['attributes']})"
+                )
+                wf = db.get(WorkflowV2, wft.workflow_id)
+                proj = db.get(ProjectV2, wf.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{wf.id=}, "
+                    f"{wf.name=}, "
+                    f"{wft.task.name=}"
+                )
+            wft.input_filters = None
+            flag_modified(wft, "input_filters")
+            WorkflowTaskReadV2(
+                **wft.model_dump(),
+                task=TaskReadV2(**wft.task.model_dump()),
+            )
+            db.add(wft)
+            logger.info(f"WorkflowTaskV2[{wft.id}] END - fixed filters")
+
+        logger.info("------ switch from workflowtasks to jobs ------")
+
+        # JOBS V2
+        stm = select(JobV2).order_by(JobV2.id)
+        jobs = db.execute(stm).scalars().all()
+        for job in jobs:
+            logger.info(f"JobV2[{job.id}] START")
+            if "filters" not in job.dataset_dump.keys():
+                logger.info(f"JobV2[{job.id}] SKIP")
+                continue
+            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
+                "types"
+            ]
+            (
+                job.dataset_dump["attribute_filters"],
+                warning,
+            ) = dict_values_to_list(
+                job.dataset_dump["filters"]["attributes"],
+                f"JobV2[{job.id}].dataset_dump.filters.attributes",
+            )
+            if warning and job.project_id is not None:
+                proj = db.get(ProjectV2, job.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{job.id=}, "
+                    f"{job.start_timestamp=}, "
+                    f"{job.end_timestamp=}, "
+                    f"{job.dataset_id=}, "
+                    f"{job.workflow_id=}."
+                )
+            job.dataset_dump.pop("filters")
+            flag_modified(job, "dataset_dump")
+            JobReadV2(**job.model_dump())
+            db.add(job)
+            logger.info(f"JobV2[{job.id}] END - fixed filters")
+
+        db.commit()
+        logger.info("Changes committed.")
+
+    logger.info("END execution of fix_db function")

fractal_server/images/__init__.py

@@ -1,4 +1,3 @@
-from .models import Filters  # noqa: F401
 from .models import SingleImage  # noqa: F401
 from .models import SingleImageTaskOutput  # noqa: F401
 from .models import SingleImageUpdate  # noqa: F401

fractal_server/images/models.py

@@ -3,15 +3,16 @@ from typing import Optional
 from typing import Union
 
 from pydantic import BaseModel
-from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
-from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.app.schemas._validators import valdict_keys
 from fractal_server.urls import normalize_url
 
+AttributeFiltersType = dict[str, list[Any]]
 
-class SingleImageBase(BaseModel):
+
+class _SingleImageBase(BaseModel):
     """
     Base for SingleImage and SingleImageTaskOutput.
 
@@ -30,9 +31,9 @@ class SingleImageBase(BaseModel):
 
     # Validators
     _attributes = validator("attributes", allow_reuse=True)(
-        valdictkeys("attributes")
+        valdict_keys("attributes")
     )
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
+    _types = validator("types", allow_reuse=True)(valdict_keys("types"))
 
     @validator("zarr_url")
     def normalize_zarr_url(cls, v: str) -> str:
@@ -44,7 +45,7 @@ class SingleImageBase(BaseModel):
             return normalize_url(v)
 
 
-class SingleImageTaskOutput(SingleImageBase):
+class SingleImageTaskOutput(_SingleImageBase):
     """
     `SingleImageBase`, with scalar `attributes` values (`None` included).
     """
@@ -63,7 +64,7 @@ class SingleImageTaskOutput(SingleImageBase):
         return v
 
 
-class SingleImage(SingleImageBase):
+class SingleImage(_SingleImageBase):
     """
     `SingleImageBase`, with scalar `attributes` values (`None` excluded).
     """
@@ -83,8 +84,8 @@ class SingleImage(SingleImageBase):
 
 class SingleImageUpdate(BaseModel):
     zarr_url: str
-    attributes: Optional[dict[str, Any]]
-    types: Optional[dict[str, bool]]
+    attributes: Optional[dict[str, Any]] = None
+    types: Optional[dict[str, bool]] = None
 
     @validator("zarr_url")
     def normalize_zarr_url(cls, v: str) -> str:
@@ -96,7 +97,7 @@ class SingleImageUpdate(BaseModel):
     ) -> dict[str, Union[int, float, str, bool]]:
         if v is not None:
             # validate keys
-            valdictkeys("attributes")(v)
+            valdict_keys("attributes")(v)
             # validate values
             for key, value in v.items():
                 if not isinstance(value, (int, float, str, bool)):
@@ -107,28 +108,4 @@ class SingleImageUpdate(BaseModel):
                     )
         return v
 
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
-
-
-class Filters(BaseModel, extra=Extra.forbid):
-    attributes: dict[str, Any] = Field(default_factory=dict)
-    types: dict[str, bool] = Field(default_factory=dict)
-
-    # Validators
-    _attributes = validator("attributes", allow_reuse=True)(
-        valdictkeys("attributes")
-    )
-    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
-
-    @validator("attributes")
-    def validate_attributes(
-        cls, v: dict[str, Any]
-    ) -> dict[str, Union[int, float, str, bool, None]]:
-        for key, value in v.items():
-            if not isinstance(value, (int, float, str, bool, type(None))):
-                raise ValueError(
-                    f"Filters.attributes[{key}] must be a scalar "
-                    "(int, float, str, bool, or None). "
-                    f"Given {value} ({type(value)})"
-                )
-        return v
+    _types = validator("types", allow_reuse=True)(valdict_keys("types"))

fractal_server/images/tools.py

@@ -4,8 +4,7 @@ from typing import Literal
 from typing import Optional
 from typing import Union
 
-from fractal_server.images import Filters
-
+from fractal_server.images.models import AttributeFiltersType
 
 ImageSearch = dict[Literal["image", "index"], Union[int, dict[str, Any]]]
 
@@ -33,52 +32,92 @@ def find_image_by_zarr_url(
     return dict(image=copy(images[ind]), index=ind)
 
 
-def match_filter(image: dict[str, Any], filters: Filters) -> bool:
+def match_filter(
+    *,
+    image: dict[str, Any],
+    type_filters: dict[str, bool],
+    attribute_filters: AttributeFiltersType,
+) -> bool:
     """
     Find whether an image matches a filter set.
 
     Arguments:
         image: A single image.
-        filters: A set of filters.
+        type_filters:
+        attribute_filters:
 
     Returns:
         Whether the image matches the filter set.
     """
+
     # Verify match with types (using a False default)
-    for key, value in filters.types.items():
+    for key, value in type_filters.items():
         if image["types"].get(key, False) != value:
             return False
-    # Verify match with attributes (only for non-None filters)
-    for key, value in filters.attributes.items():
-        if value is None:
-            continue
-        if image["attributes"].get(key) != value:
+
+    # Verify match with attributes (only for not-None filters)
+    for key, values in attribute_filters.items():
+        if image["attributes"].get(key) not in values:
             return False
+
     return True
 
 
 def filter_image_list(
     images: list[dict[str, Any]],
-    filters: Filters,
+    type_filters: Optional[dict[str, bool]] = None,
+    attribute_filters: Optional[AttributeFiltersType] = None,
 ) -> list[dict[str, Any]]:
     """
     Compute a sublist with images that match a filter set.
 
     Arguments:
         images: A list of images.
-        filters: A set of filters.
+        type_filters:
+        attribute_filters:
 
     Returns:
         List of the `images` elements which match the filter set.
     """
 
     # When no filter is provided, return all images
-    if filters.attributes == {} and filters.types == {}:
+    if type_filters is None and attribute_filters is None:
         return images
+    actual_type_filters = type_filters or {}
+    actual_attribute_filters = attribute_filters or {}
 
     filtered_images = [
         copy(this_image)
         for this_image in images
-        if match_filter(this_image, filters=filters)
+        if match_filter(
+            image=this_image,
+            type_filters=actual_type_filters,
+            attribute_filters=actual_attribute_filters,
+        )
     ]
     return filtered_images
+
+
+def merge_type_filters(
+    *,
+    task_input_types: dict[str, bool],
+    wftask_type_filters: dict[str, bool],
+) -> dict[str, bool]:
+    """
+    Merge two type-filters sets, if they are compatible.
+    """
+    all_keys = set(task_input_types.keys()) | set(wftask_type_filters.keys())
+    for key in all_keys:
+        if (
+            key in task_input_types.keys()
+            and key in wftask_type_filters.keys()
+            and task_input_types[key] != wftask_type_filters[key]
+        ):
+            raise ValueError(
+                "Cannot merge type filters "
+                f"`{task_input_types}` (from task) "
+                f"and `{wftask_type_filters}` (from workflowtask)."
+            )
+    merged_dict = task_input_types
+    merged_dict.update(wftask_type_filters)
+    return merged_dict

fractal_server/migrations/versions/db09233ad13a_split_filters_and_keep_old_columns.py

@@ -0,0 +1,96 @@
+"""split filters and keep old columns
+
+Revision ID: db09233ad13a
+Revises: 316140ff7ee1
+Create Date: 2025-01-14 14:50:46.007222
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "db09233ad13a"
+down_revision = "316140ff7ee1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                sa.JSON(),
+                server_default="{}",
+                nullable=False,
+            )
+        )
+        batch_op.alter_column(
+            "filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+            server_default="null",
+        )
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "attribute_filters",
+                sa.JSON(),
+                server_default="{}",
+                nullable=False,
+            )
+        )
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type_filters", sa.JSON(), server_default="{}", nullable=False
+            )
+        )
+        batch_op.alter_column(
+            "input_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+            server_default="null",
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "input_filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+            existing_server_default=sa.text(
+                '\'{"attributes": {}, "types": {}}\'::json'
+            ),
+        )
+        batch_op.drop_column("type_filters")
+
+    with op.batch_alter_table("jobv2", schema=None) as batch_op:
+        batch_op.drop_column("attribute_filters")
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "filters",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+            existing_server_default=sa.text(
+                '\'{"attributes": {}, "types": {}}\'::json'
+            ),
+        )
+        batch_op.drop_column("attribute_filters")
+        batch_op.drop_column("type_filters")
+
+    # ### end Alembic commands ###

fractal_server/utils.py

@@ -107,18 +107,20 @@ def execute_command_sync(
     returncode = res.returncode
     stdout = res.stdout
     stderr = res.stderr
-    logger.debug(f"{returncode=}")
-    logger.debug("STDOUT:")
-    logger.debug(stdout)
-    logger.debug("STDERR:")
-    logger.debug(stderr)
     if res.returncode != 0:
         logger.debug(f"ERROR in subprocess call to '{command}'")
         raise RuntimeError(
             f"Command {command} failed.\n"
             f"returncode={res.returncode}\n"
-            f"{stdout=}\n"
-            f"{stderr=}\n"
+            "STDOUT:\n"
+            f"{stdout}\n"
+            "STDERR:\n"
+            f"{stderr}\n"
         )
+    logger.debug(f"{returncode=}")
+    logger.debug("STDOUT:")
+    logger.debug(stdout)
+    logger.debug("STDERR:")
+    logger.debug(stderr)
     logger.debug(f"END   subprocess call to '{command}'")
     return stdout