fractal-server 2.11.0a5__tar.gz → 2.11.0a7__tar.gz

{fractal_server-2.11.0a5 → fractal_server-2.11.0a7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.11.0a5
+Version: 2.11.0a7
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

fractal_server-2.11.0a7/fractal_server/__init__.py

@@ -0,0 +1 @@
+__VERSION__ = "2.11.0a7"

{fractal_server-2.11.0a5 → fractal_server-2.11.0a7}/fractal_server/app/schemas/_filter_validators.py

@@ -22,14 +22,10 @@ def validate_attribute_filters(
 
     attribute_filters = valdict_keys("attribute_filters")(attribute_filters)
     for key, values in attribute_filters.items():
-        if values is None:
-            # values=None corresponds to not applying any filter for
-            # attribute `key`
-            pass
-        elif values == []:
-            # WARNING: in this case, no image can match with the current
-            # filter. In the future we may deprecate this possibility.
-            pass
+        if values == []:
+            raise ValueError(
+                f"attribute_filters[{key}] cannot be an empty list."
+            )
         else:
             # values is a non-empty list, and its items must all be of the
             # same scalar non-None type

fractal_server-2.11.0a7/fractal_server/data_migrations/2_11_0.py

@@ -0,0 +1,159 @@
+import logging
+from typing import Union
+
+from sqlalchemy.orm.attributes import flag_modified
+from sqlmodel import select
+
+from fractal_server.app.db import get_sync_db
+from fractal_server.app.models import DatasetV2
+from fractal_server.app.models import JobV2
+from fractal_server.app.models import ProjectV2
+from fractal_server.app.models import WorkflowTaskV2
+from fractal_server.app.models import WorkflowV2
+from fractal_server.app.schemas.v2 import DatasetReadV2
+from fractal_server.app.schemas.v2 import JobReadV2
+from fractal_server.app.schemas.v2 import ProjectReadV2
+from fractal_server.app.schemas.v2 import TaskReadV2
+from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
+from fractal_server.images.models import AttributeFiltersType
+
+logger = logging.getLogger("fix_db")
+logger.setLevel(logging.INFO)
+
+
+def dict_values_to_list(
+    input_dict: dict[str, Union[int, float, bool, str, None]],
+    identifier: str,
+) -> tuple[AttributeFiltersType, bool]:
+    was_there_a_warning = False
+    for k, v in input_dict.items():
+        if not isinstance(v, (int, float, bool, str, type(None))):
+            error_msg = (
+                f"Attribute '{k}' from '{identifier}' "
+                "has invalid type '{type(v)}'."
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
+        elif v is None:
+            logger.warning(
+                f"Attribute '{k}' from '{identifier}' is "
+                "None and it will be removed."
+            )
+            was_there_a_warning = True
+        else:
+            input_dict[k] = [v]
+    return input_dict, was_there_a_warning
+
+
+def fix_db():
+    logger.info("START execution of fix_db function")
+
+    with next(get_sync_db()) as db:
+        # DatasetV2.filters
+        stm = select(DatasetV2).order_by(DatasetV2.id)
+        datasets = db.execute(stm).scalars().all()
+        for ds in datasets:
+            logger.info(f"DatasetV2[{ds.id}] START")
+            ds.attribute_filters, warning = dict_values_to_list(
+                ds.filters["attributes"],
+                f"Dataset[{ds.id}].filters.attributes",
+            )
+            if warning:
+                proj = db.get(ProjectV2, ds.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{ds.id=}, "
+                    f"{ds.name=}"
+                )
+            ds.type_filters = ds.filters["types"]
+            ds.filters = None
+            for i, h in enumerate(ds.history):
+                ds.history[i]["workflowtask"]["type_filters"] = h[
+                    "workflowtask"
+                ]["input_filters"]["types"]
+            flag_modified(ds, "history")
+            DatasetReadV2(
+                **ds.model_dump(),
+                project=ProjectReadV2(**ds.project.model_dump()),
+            )
+            db.add(ds)
+            logger.info(f"DatasetV2[{ds.id}] END - fixed filters")
+
+        logger.info("------ switch from dataset to workflowtasks ------")
+
+        # WorkflowTaskV2.input_filters
+        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
+        wftasks = db.execute(stm).scalars().all()
+        for wft in wftasks:
+            logger.info(f"WorkflowTaskV2[{wft.id}] START")
+            wft.type_filters = wft.input_filters["types"]
+            if wft.input_filters["attributes"]:
+                logger.warning(
+                    "Removing input_filters['attributes']. "
+                    f"(previous value: {wft.input_filters['attributes']})"
+                )
+                wf = db.get(WorkflowV2, wft.workflow_id)
+                proj = db.get(ProjectV2, wf.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{wf.id=}, "
+                    f"{wf.name=}, "
+                    f"{wft.task.name=}"
+                )
+            wft.input_filters = None
+            flag_modified(wft, "input_filters")
+            WorkflowTaskReadV2(
+                **wft.model_dump(),
+                task=TaskReadV2(**wft.task.model_dump()),
+            )
+            db.add(wft)
+            logger.info(f"WorkflowTaskV2[{wft.id}] END - fixed filters")
+
+        logger.info("------ switch from workflowtasks to jobs ------")
+
+        # JOBS V2
+        stm = select(JobV2).order_by(JobV2.id)
+        jobs = db.execute(stm).scalars().all()
+        for job in jobs:
+            logger.info(f"JobV2[{job.id}] START")
+            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
+                "types"
+            ]
+            (
+                job.dataset_dump["attribute_filters"],
+                warning,
+            ) = dict_values_to_list(
+                job.dataset_dump["filters"]["attributes"],
+                f"JobV2[{job.id}].dataset_dump.filters.attributes",
+            )
+            if warning and job.project_id is not None:
+                proj = db.get(ProjectV2, job.project_id)
+                logger.warning(
+                    "Additional information: "
+                    f"{proj.id=}, "
+                    f"{proj.name=}, "
+                    f"{proj.user_list[0].email=}, "
+                    f"{job.id=}, "
+                    f"{job.start_timestamp=}, "
+                    f"{job.end_timestamp=}, "
+                    f"{job.dataset_id=}, "
+                    f"{job.workflow_id=}."
+                )
+                # FIXME
+                pass
+            job.dataset_dump.pop("filters")
+            flag_modified(job, "dataset_dump")
+            JobReadV2(**job.model_dump())
+            db.add(job)
+            logger.info(f"JobV2[{job.id}] END - fixed filters")
+
+        db.commit()
+        logger.info("Changes committed.")
+
+    logger.info("END execution of fix_db function")

{fractal_server-2.11.0a5 → fractal_server-2.11.0a7}/fractal_server/images/models.py

@@ -9,7 +9,7 @@ from pydantic import validator
 from fractal_server.app.schemas._validators import valdict_keys
 from fractal_server.urls import normalize_url
 
-AttributeFiltersType = dict[str, Optional[list[Any]]]
+AttributeFiltersType = dict[str, list[Any]]
 
 
 class _SingleImageBase(BaseModel):

{fractal_server-2.11.0a5 → fractal_server-2.11.0a7}/fractal_server/images/tools.py

@@ -57,8 +57,6 @@ def match_filter(
 
     # Verify match with attributes (only for not-None filters)
     for key, values in attribute_filters.items():
-        if values is None:
-            continue
         if image["attributes"].get(key) not in values:
             return False
 

{fractal_server-2.11.0a5 → fractal_server-2.11.0a7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fractal-server"
-version = "2.11.0a5"
+version = "2.11.0a7"
 description = "Backend component of the Fractal analytics platform"
 authors = [
     { name="Tommaso Comparin", email="tommaso.comparin@exact-lab.it" },
@@ -95,7 +95,7 @@ filterwarnings = [
 ]
 
 [tool.bumpver]
-current_version = "2.11.0a5"
+current_version = "2.11.0a7"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

fractal_server-2.11.0a5/fractal_server/__init__.py

@@ -1 +0,0 @@
-__VERSION__ = "2.11.0a5"

fractal_server-2.11.0a5/fractal_server/data_migrations/2_11_0.py

@@ -1,69 +0,0 @@
-import logging
-
-from sqlalchemy.orm.attributes import flag_modified
-from sqlmodel import select
-
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.models import DatasetV2
-from fractal_server.app.models import JobV2
-from fractal_server.app.models import WorkflowTaskV2
-
-logger = logging.getLogger("fix_db")
-logger.setLevel(logging.INFO)
-
-
-def fix_db():
-
-    logger.info("START execution of fix_db function")
-
-    with next(get_sync_db()) as db:
-
-        # DatasetV2.filters
-        # DatasetV2.history[].workflowtask.input_filters
-        stm = select(DatasetV2).order_by(DatasetV2.id)
-        datasets = db.execute(stm).scalars().all()
-        for ds in datasets:
-            ds.attribute_filters = ds.filters["attributes"]
-            ds.type_filters = ds.filters["types"]
-            ds.filters = None
-            for i, h in enumerate(ds.history):
-                ds.history[i]["workflowtask"]["type_filters"] = h[
-                    "workflowtask"
-                ]["input_filters"]["types"]
-            flag_modified(ds, "history")
-            db.add(ds)
-            logger.info(f"Fixed filters in DatasetV2[{ds.id}]")
-
-        # WorkflowTaskV2.input_filters
-        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
-        wftasks = db.execute(stm).scalars().all()
-        for wft in wftasks:
-            wft.type_filters = wft.input_filters["types"]
-            if wft.input_filters["attributes"]:
-                logger.warning(
-                    f"Removing WorkflowTaskV2[{wft.id}].input_filters"
-                    f"['attributes'] = {wft.input_filters['attributes']}"
-                )
-            wft.input_filters = None
-            flag_modified(wft, "input_filters")
-            db.add(wft)
-            logger.info(f"Fixed filters in WorkflowTaskV2[{wft.id}]")
-
-        # JOBS V2
-        stm = select(JobV2).order_by(JobV2.id)
-        jobs = db.execute(stm).scalars().all()
-        for job in jobs:
-            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
-                "types"
-            ]
-            job.dataset_dump["attribute_filters"] = job.dataset_dump[
-                "filters"
-            ]["attributes"]
-            job.dataset_dump.pop("filters")
-            flag_modified(job, "dataset_dump")
-            logger.info(f"Fixed filters in JobV2[{job.id}].datasetdump")
-
-        db.commit()
-        logger.info("Changes committed.")
-
-    logger.info("END execution of fix_db function")