fractal-server 2.10.5__py3-none-any.whl → 2.11.0a2__py3-none-any.whl

fractal_server/app/runner/v2/runner.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
 from copy import copy
@@ -7,27 +6,28 @@ from pathlib import Path
 from typing import Callable
 from typing import Optional
 
-from ....images import Filters
+from sqlalchemy.orm.attributes import flag_modified
+
 from ....images import SingleImage
 from ....images.tools import filter_image_list
 from ....images.tools import find_image_by_zarr_url
 from ....images.tools import match_filter
 from ..exceptions import JobExecutionError
-from ..filenames import FILTERS_FILENAME
-from ..filenames import HISTORY_FILENAME
-from ..filenames import IMAGES_FILENAME
 from .runner_functions import no_op_submit_setup_call
 from .runner_functions import run_v2_task_compound
 from .runner_functions import run_v2_task_non_parallel
 from .runner_functions import run_v2_task_parallel
 from .task_interface import TaskOutput
+from fractal_server.app.db import get_sync_db
 from fractal_server.app.models.v2 import DatasetV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.schemas.v2.dataset import _DatasetHistoryItemV2
 from fractal_server.app.schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
+from fractal_server.images.models import AttributeFiltersType
 
 
 def execute_tasks_v2(
+    *,
     wf_task_list: list[WorkflowTaskV2],
     dataset: DatasetV2,
     executor: ThreadPoolExecutor,
@@ -35,20 +35,21 @@ def execute_tasks_v2(
     workflow_dir_remote: Optional[Path] = None,
     logger_name: Optional[str] = None,
     submit_setup_call: Callable = no_op_submit_setup_call,
-) -> DatasetV2:
-
+    job_attribute_filters: AttributeFiltersType,
+) -> None:
     logger = logging.getLogger(logger_name)
 
-    if (
-        not workflow_dir_local.exists()
-    ):  # FIXME: this should have already happened
+    if not workflow_dir_local.exists():
+        logger.warning(
+            f"Now creating {workflow_dir_local}, "
+            "but it should have already happened."
+        )
         workflow_dir_local.mkdir()
 
     # Initialize local dataset attributes
     zarr_dir = dataset.zarr_dir
     tmp_images = deepcopy(dataset.images)
-    tmp_filters = deepcopy(dataset.filters)
-    tmp_history = []
+    tmp_type_filters = deepcopy(dataset.type_filters)
 
     for wftask in wf_task_list:
         task = wftask.task
@@ -58,26 +59,38 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Get filtered images
-        pre_filters = dict(
-            types=copy(tmp_filters["types"]),
-            attributes=copy(tmp_filters["attributes"]),
-        )
-        pre_filters["types"].update(wftask.input_filters["types"])
-        pre_filters["attributes"].update(wftask.input_filters["attributes"])
+        pre_type_filters = copy(tmp_type_filters)
+        pre_type_filters.update(wftask.type_filters)
         filtered_images = filter_image_list(
             images=tmp_images,
-            filters=Filters(**pre_filters),
+            type_filters=pre_type_filters,
+            attribute_filters=job_attribute_filters,
         )
         # Verify that filtered images comply with task input_types
         for image in filtered_images:
-            if not match_filter(image, Filters(types=task.input_types)):
+            if not match_filter(
+                image=image,
+                type_filters=task.input_types,
+                attribute_filters={},
+            ):
                 raise JobExecutionError(
                     "Invalid filtered image list\n"
                     f"Task input types: {task.input_types=}\n"
                     f'Image zarr_url: {image["zarr_url"]}\n'
                     f'Image types: {image["types"]}\n'
                 )
-
+        # First, set status SUBMITTED in dataset.history for each wftask
+        with next(get_sync_db()) as db:
+            db_dataset = db.get(DatasetV2, dataset.id)
+            new_history_item = _DatasetHistoryItemV2(
+                workflowtask=wftask,
+                status=WorkflowTaskStatusTypeV2.SUBMITTED,
+                parallelization=dict(),  # FIXME: re-include parallelization
+            ).dict()
+            db_dataset.history.append(new_history_item)
+            flag_modified(db_dataset, "history")
+            db.merge(db_dataset)
+            db.commit()
         # TASK EXECUTION (V2)
         if task.type == "non_parallel":
             current_task_output = run_v2_task_non_parallel(
@@ -249,69 +262,47 @@ def execute_tasks_v2(
             else:
                 tmp_images.pop(img_search["index"])
 
-        # Update filters.attributes:
-        # current + (task_output: not really, in current examples..)
-        if current_task_output.filters is not None:
-            tmp_filters["attributes"].update(
-                current_task_output.filters.attributes
-            )
-
-        # Find manifest ouptut types
-        types_from_manifest = task.output_types
+        # Update type_filters
 
-        # Find task-output types
-        if current_task_output.filters is not None:
-            types_from_task = current_task_output.filters.types
-        else:
-            types_from_task = {}
+        # Assign the type filters based on different sources
+        # (task manifest and post-execution task output)
+        type_filters_from_task_manifest = task.output_types
+        type_filters_from_task_output = current_task_output.type_filters
 
         # Check that key sets are disjoint
-        set_types_from_manifest = set(types_from_manifest.keys())
-        set_types_from_task = set(types_from_task.keys())
-        if not set_types_from_manifest.isdisjoint(set_types_from_task):
-            overlap = set_types_from_manifest.intersection(set_types_from_task)
+        keys_from_manifest = set(type_filters_from_task_manifest.keys())
+        keys_from_task_output = set(type_filters_from_task_output.keys())
+        if not keys_from_manifest.isdisjoint(keys_from_task_output):
+            overlap = keys_from_manifest.intersection(keys_from_task_output)
             raise JobExecutionError(
                 "Some type filters are being set twice, "
                 f"for task '{task_name}'.\n"
-                f"Types from task output: {types_from_task}\n"
-                f"Types from task maniest: {types_from_manifest}\n"
+                f"Types from task output: {type_filters_from_task_output}\n"
+                "Types from task manifest: "
+                f"{type_filters_from_task_manifest}\n"
                 f"Overlapping keys: {overlap}"
             )
 
         # Update filters.types
-        tmp_filters["types"].update(types_from_manifest)
-        tmp_filters["types"].update(types_from_task)
+        tmp_type_filters.update(type_filters_from_task_manifest)
+        tmp_type_filters.update(type_filters_from_task_output)
 
-        # Update history (based on _DatasetHistoryItemV2)
-        history_item = _DatasetHistoryItemV2(
-            workflowtask=wftask,
-            status=WorkflowTaskStatusTypeV2.DONE,
-            parallelization=dict(
-                # task_type=wftask.task.type,  # FIXME: breaks for V1 tasks
-                # component_list=fil, #FIXME
-            ),
-        ).dict()
-        tmp_history.append(history_item)
-
-        # Write current dataset attributes (history, images, filters) into
-        # temporary files which can be used (1) to retrieve the latest state
+        # Write current dataset attributes (history, images, filters) into the
+        # database. They can be used (1) to retrieve the latest state
         # when the job fails, (2) from within endpoints that need up-to-date
         # information
-        with open(workflow_dir_local / HISTORY_FILENAME, "w") as f:
-            json.dump(tmp_history, f, indent=2)
-        with open(workflow_dir_local / FILTERS_FILENAME, "w") as f:
-            json.dump(tmp_filters, f, indent=2)
-        with open(workflow_dir_local / IMAGES_FILENAME, "w") as f:
-            json.dump(tmp_images, f, indent=2)
+        with next(get_sync_db()) as db:
+            db_dataset = db.get(DatasetV2, dataset.id)
+            db_dataset.history[-1]["status"] = WorkflowTaskStatusTypeV2.DONE
+            db_dataset.type_filters = tmp_type_filters
+            db_dataset.images = tmp_images
+            for attribute_name in [
+                "type_filters",
+                "history",
+                "images",
+            ]:
+                flag_modified(db_dataset, attribute_name)
+            db.merge(db_dataset)
+            db.commit()
 
         logger.debug(f'END    {wftask.order}-th task (name="{task_name}")')
-
-    # NOTE: tmp_history only contains the newly-added history items (to be
-    # appended to the original history), while tmp_filters and tmp_images
-    # represent the new attributes (to replace the original ones)
-    result = dict(
-        history=tmp_history,
-        filters=tmp_filters,
-        images=tmp_images,
-    )
-    return result

fractal_server/app/runner/v2/task_interface.py

@@ -1,22 +1,47 @@
 from typing import Any
+from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
 from ....images import SingleImageTaskOutput
-from fractal_server.images import Filters
+from fractal_server.app.schemas._filter_validators import validate_type_filters
+from fractal_server.app.schemas._validators import root_validate_dict_keys
 from fractal_server.urls import normalize_url
 
 
+class LegacyFilters(BaseModel, extra=Extra.forbid):
+    """
+    For fractal-server<2.11, task output could include both
+    `filters["attributes"]` and `filters["types"]`. In the new version
+    there is a single field, named `type_filters`.
+    The current schema is only used to convert old type filters into the
+    new form, but it will reject any attribute filters.
+    """
+
+    types: dict[str, bool] = Field(default_factory=dict)
+    _types = validator("types", allow_reuse=True)(validate_type_filters)
+
+
 class TaskOutput(BaseModel, extra=Extra.forbid):
 
     image_list_updates: list[SingleImageTaskOutput] = Field(
         default_factory=list
     )
     image_list_removals: list[str] = Field(default_factory=list)
-    filters: Filters = Field(default_factory=Filters)
+
+    filters: Optional[LegacyFilters] = None
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
 
     def check_zarr_urls_are_unique(self) -> None:
         zarr_urls = [img.zarr_url for img in self.image_list_updates]
@@ -37,6 +62,20 @@ class TaskOutput(BaseModel, extra=Extra.forbid):
                 msg = f"{msg}\n{duplicate}"
             raise ValueError(msg)
 
+    @root_validator()
+    def update_legacy_filters(cls, values):
+        if values["filters"] is not None:
+            if values["type_filters"] != {}:
+                raise ValueError(
+                    "Cannot set both (legacy) 'filters' and 'type_filters'."
+                )
+            else:
+                # Convert legacy filters.types into new type_filters
+                values["type_filters"] = values["filters"].types
+                values["filters"] = None
+
+        return values
+
     @validator("image_list_removals")
     def normalize_paths(cls, v: list[str]) -> list[str]:
         return [normalize_url(zarr_url) for zarr_url in v]

fractal_server/app/schemas/_filter_validators.py

@@ -0,0 +1,47 @@
+from typing import Optional
+
+from ._validators import valdict_keys
+from fractal_server.images.models import AttributeFiltersType
+
+
+def validate_type_filters(
+    type_filters: Optional[dict[str, bool]]
+) -> dict[str, bool]:
+    if type_filters is None:
+        raise ValueError("'type_filters' cannot be 'None'.")
+
+    type_filters = valdict_keys("type_filters")(type_filters)
+    return type_filters
+
+
+def validate_attribute_filters(
+    attribute_filters: Optional[AttributeFiltersType],
+) -> AttributeFiltersType:
+    if attribute_filters is None:
+        raise ValueError("'attribute_filters' cannot be 'None'.")
+
+    attribute_filters = valdict_keys("attribute_filters")(attribute_filters)
+    for key, values in attribute_filters.items():
+        if values is None:
+            # values=None corresponds to not applying any filter for
+            # attribute `key`
+            pass
+        elif values == []:
+            # WARNING: in this case, no image can match with the current
+            # filter. In the future we may deprecate this possibility.
+            pass
+        else:
+            # values is a non-empty list, and its items must all be of the
+            # same scalar non-None type
+            _type = type(values[0])
+            if not all(isinstance(value, _type) for value in values):
+                raise ValueError(
+                    f"attribute_filters[{key}] has values with "
+                    f"non-homogeneous types: {values}."
+                )
+            if _type not in (int, float, str, bool):
+                raise ValueError(
+                    f"attribute_filters[{key}] has values with "
+                    f"invalid types: {values}."
+                )
+    return attribute_filters

fractal_server/app/schemas/_validators.py

@@ -27,7 +27,7 @@ def valstr(attribute: str, accept_none: bool = False):
     return val
 
 
-def valdictkeys(attribute: str):
+def valdict_keys(attribute: str):
     def val(d: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]:
         """
         Apply valstr to every key of the dictionary, and fail if there are
@@ -38,7 +38,7 @@ def valdictkeys(attribute: str):
             new_keys = [valstr(f"{attribute}[{key}]")(key) for key in old_keys]
             if len(new_keys) != len(set(new_keys)):
                 raise ValueError(
-                    f"Dictionary contains multiple identical keys: {d}."
+                    f"Dictionary contains multiple identical keys: '{d}'."
                 )
             for old_key, new_key in zip(old_keys, new_keys):
                 if new_key != old_key:
@@ -101,3 +101,14 @@ def val_unique_list(attribute: str):
         return must_be_unique
 
     return val
+
+
+def root_validate_dict_keys(cls, object: dict) -> dict:
+    """
+    For each dictionary in `object.values()`,
+    checks that that dictionary has only keys of type str.
+    """
+    for dictionary in (v for v in object.values() if isinstance(v, dict)):
+        if not all(isinstance(key, str) for key in dictionary.keys()):
+            raise ValueError("Dictionary keys must be strings.")
+    return object

fractal_server/app/schemas/v2/dataset.py

@@ -1,17 +1,22 @@
 from datetime import datetime
+from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 
+from .._filter_validators import validate_attribute_filters
+from .._filter_validators import validate_type_filters
+from .._validators import root_validate_dict_keys
 from .._validators import valstr
 from .dumps import WorkflowTaskDumpV2
 from .project import ProjectReadV2
 from .workflowtask import WorkflowTaskStatusTypeV2
-from fractal_server.images import Filters
 from fractal_server.images import SingleImage
+from fractal_server.images.models import AttributeFiltersType
 from fractal_server.urls import normalize_url
 
 
@@ -34,17 +39,29 @@ class DatasetCreateV2(BaseModel, extra=Extra.forbid):
 
     zarr_dir: Optional[str] = None
 
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
     # Validators
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             return normalize_url(v)
         return v
 
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-
 
 class DatasetReadV2(BaseModel):
 
@@ -59,24 +76,37 @@ class DatasetReadV2(BaseModel):
     timestamp_created: datetime
 
     zarr_dir: str
-    filters: Filters = Field(default_factory=Filters)
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType
 
 
 class DatasetUpdateV2(BaseModel, extra=Extra.forbid):
 
     name: Optional[str]
     zarr_dir: Optional[str]
-    filters: Optional[Filters]
+    type_filters: Optional[dict[str, bool]]
+    attribute_filters: Optional[dict[str, list[Any]]]
 
     # Validators
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: Optional[str]) -> Optional[str]:
         if v is not None:
             return normalize_url(v)
         return v
 
-    _name = validator("name", allow_reuse=True)(valstr("name"))
-
 
 class DatasetImportV2(BaseModel, extra=Extra.forbid):
     """
@@ -86,15 +116,29 @@ class DatasetImportV2(BaseModel, extra=Extra.forbid):
         name:
         zarr_dir:
         images:
-        filters:
+        type_filters:
+        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage] = Field(default_factory=list)
-    filters: Filters = Field(default_factory=Filters)
+
+    type_filters: dict[str, bool] = Field(default_factory=dict)
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
 
     # Validators
+
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _type_filters = validator("type_filters", allow_reuse=True)(
+        validate_type_filters
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
+
     @validator("zarr_dir")
     def normalize_zarr_dir(cls, v: str) -> str:
         return normalize_url(v)
@@ -108,10 +152,12 @@ class DatasetExportV2(BaseModel):
         name:
         zarr_dir:
         images:
-        filters:
+        type_filters:
+        attribute_filters:
     """
 
     name: str
     zarr_dir: str
     images: list[SingleImage]
-    filters: Filters
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/dumps.py

@@ -13,7 +13,7 @@ from typing import Optional
 from pydantic import BaseModel
 from pydantic import Extra
 
-from fractal_server.images import Filters
+from fractal_server.images.models import AttributeFiltersType
 
 
 class ProjectDumpV2(BaseModel, extra=Extra.forbid):
@@ -39,19 +39,16 @@ class TaskDumpV2(BaseModel):
 
 class WorkflowTaskDumpV2(BaseModel):
     """
-    Before v2.5.0, WorkflowTaskV2 could have `task_id=task=None` and
-    non-`None` `task_legacy_id` and `task_legacy`. Since these objects
-    may still exist in the database after version updates, we are setting
-    `task_id` and `task` to `Optional` to avoid response-validation errors
+    We do not include 'extra=Extra.forbid' because legacy data may include
+    'input_filters' field and we want to avoid response-validation errors
     for the endpoints that GET datasets.
-    Ref issue #1783.
     """
 
     id: int
     workflow_id: int
     order: Optional[int]
 
-    input_filters: Filters
+    type_filters: dict[str, bool]
 
     task_id: Optional[int]
     task: Optional[TaskDumpV2]
@@ -71,4 +68,5 @@ class DatasetDumpV2(BaseModel, extra=Extra.forbid):
     timestamp_created: str
 
     zarr_dir: str
-    filters: Filters
+    type_filters: dict[str, bool]
+    attribute_filters: AttributeFiltersType

fractal_server/app/schemas/v2/job.py

@@ -4,13 +4,18 @@ from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
+from pydantic import Field
+from pydantic import root_validator
 from pydantic import validator
 from pydantic.types import StrictStr
 
+from .._filter_validators import validate_attribute_filters
+from .._validators import root_validate_dict_keys
 from .._validators import valstr
 from .dumps import DatasetDumpV2
 from .dumps import ProjectDumpV2
 from .dumps import WorkflowDumpV2
+from fractal_server.images.models import AttributeFiltersType
 
 
 class JobStatusTypeV2(str, Enum):
@@ -41,10 +46,18 @@ class JobCreateV2(BaseModel, extra=Extra.forbid):
     slurm_account: Optional[StrictStr] = None
     worker_init: Optional[str]
 
+    attribute_filters: AttributeFiltersType = Field(default_factory=dict)
+
     # Validators
     _worker_init = validator("worker_init", allow_reuse=True)(
         valstr("worker_init")
     )
+    _dict_keys = root_validator(pre=True, allow_reuse=True)(
+        root_validate_dict_keys
+    )
+    _attribute_filters = validator("attribute_filters", allow_reuse=True)(
+        validate_attribute_filters
+    )
 
     @validator("first_task_index", always=True)
     def first_task_index_non_negative(cls, v, values):
@@ -99,6 +112,7 @@ class JobReadV2(BaseModel):
     first_task_index: Optional[int]
     last_task_index: Optional[int]
     worker_init: Optional[str]
+    attribute_filters: AttributeFiltersType
 
 
 class JobUpdateV2(BaseModel, extra=Extra.forbid):

fractal_server/app/schemas/v2/task.py

@@ -10,7 +10,7 @@ from pydantic import root_validator
 from pydantic import validator
 
 from fractal_server.app.schemas._validators import val_unique_list
-from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.app.schemas._validators import valdict_keys
 from fractal_server.app.schemas._validators import valstr
 from fractal_server.string_tools import validate_cmd
 
@@ -66,25 +66,25 @@ class TaskCreateV2(BaseModel, extra=Extra.forbid):
     _version = validator("version", allow_reuse=True)(valstr("version"))
 
     _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
-        valdictkeys("meta_non_parallel")
+        valdict_keys("meta_non_parallel")
     )
     _meta_parallel = validator("meta_parallel", allow_reuse=True)(
-        valdictkeys("meta_parallel")
+        valdict_keys("meta_parallel")
     )
     _args_schema_non_parallel = validator(
         "args_schema_non_parallel", allow_reuse=True
-    )(valdictkeys("args_schema_non_parallel"))
+    )(valdict_keys("args_schema_non_parallel"))
     _args_schema_parallel = validator(
         "args_schema_parallel", allow_reuse=True
-    )(valdictkeys("args_schema_parallel"))
+    )(valdict_keys("args_schema_parallel"))
     _args_schema_version = validator("args_schema_version", allow_reuse=True)(
         valstr("args_schema_version")
     )
     _input_types = validator("input_types", allow_reuse=True)(
-        valdictkeys("input_types")
+        valdict_keys("input_types")
     )
     _output_types = validator("output_types", allow_reuse=True)(
-        valdictkeys("output_types")
+        valdict_keys("output_types")
     )
 
     _category = validator("category", allow_reuse=True)(
@@ -158,10 +158,10 @@ class TaskUpdateV2(BaseModel, extra=Extra.forbid):
         "command_non_parallel", allow_reuse=True
     )(valstr("command_non_parallel"))
     _input_types = validator("input_types", allow_reuse=True)(
-        valdictkeys("input_types")
+        valdict_keys("input_types")
     )
     _output_types = validator("output_types", allow_reuse=True)(
-        valdictkeys("output_types")
+        valdict_keys("output_types")
     )
 
     _category = validator("category", allow_reuse=True)(

fractal_server/app/schemas/v2/task_group.py

@@ -8,7 +8,7 @@ from pydantic import Field
 from pydantic import validator
 
 from .._validators import val_absolute_path
-from .._validators import valdictkeys
+from .._validators import valdict_keys
 from .._validators import valstr
 from .task import TaskReadV2
 
@@ -57,7 +57,7 @@ class TaskGroupCreateV2(BaseModel, extra=Extra.forbid):
     )
     _pinned_package_versions = validator(
         "pinned_package_versions", allow_reuse=True
-    )(valdictkeys("pinned_package_versions"))
+    )(valdict_keys("pinned_package_versions"))
     _pip_extras = validator("pip_extras", allow_reuse=True)(
         valstr("pip_extras")
     )