fractal-server 2.11.0a3__py3-none-any.whl → 2.11.0a4__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.11.0a3"
+__VERSION__ = "2.11.0a4"

fractal_server/app/runner/executors/slurm/_slurm_config.py

@@ -423,7 +423,7 @@ def _parse_mem_value(raw_mem: Union[str, int]) -> int:
     )
 
     # Handle integer argument
-    if isinstance(raw_mem, int):
+    if type(raw_mem) is int:
         return raw_mem
 
     # Handle string argument

fractal_server/app/runner/v2/merge_outputs.py

@@ -15,13 +15,6 @@ def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
         final_image_list_updates.extend(task_output.image_list_updates)
         final_image_list_removals.extend(task_output.image_list_removals)
 
-        # Check that all type_filters are the same
-        if task_output.type_filters != task_outputs[0].type_filters:
-            raise ValueError(
-                f"{task_output.type_filters=} "
-                f"but {task_outputs[0].type_filters=}"
-            )
-
     # Note: the ordering of `image_list_removals` is not guaranteed
     final_image_list_updates = deduplicate_list(final_image_list_updates)
     final_image_list_removals = list(set(final_image_list_removals))
@@ -29,7 +22,6 @@ def merge_outputs(task_outputs: list[TaskOutput]) -> TaskOutput:
     final_output = TaskOutput(
         image_list_updates=final_image_list_updates,
         image_list_removals=final_image_list_removals,
-        type_filters=task_outputs[0].type_filters,
     )
 
     return final_output

fractal_server/app/runner/v2/runner.py

@@ -11,7 +11,6 @@ from sqlalchemy.orm.attributes import flag_modified
 from ....images import SingleImage
 from ....images.tools import filter_image_list
 from ....images.tools import find_image_by_zarr_url
-from ....images.tools import match_filter
 from ..exceptions import JobExecutionError
 from .runner_functions import no_op_submit_setup_call
 from .runner_functions import run_v2_task_compound
@@ -24,6 +23,7 @@ from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.schemas.v2.dataset import _DatasetHistoryItemV2
 from fractal_server.app.schemas.v2.workflowtask import WorkflowTaskStatusTypeV2
 from fractal_server.images.models import AttributeFiltersType
+from fractal_server.images.tools import merge_type_filters
 
 
 def execute_tasks_v2(
@@ -49,7 +49,7 @@ def execute_tasks_v2(
     # Initialize local dataset attributes
     zarr_dir = dataset.zarr_dir
     tmp_images = deepcopy(dataset.images)
-    tmp_type_filters = deepcopy(dataset.type_filters)
+    current_dataset_type_filters = deepcopy(dataset.type_filters)
 
     for wftask in wf_task_list:
         task = wftask.task
@@ -59,26 +59,18 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Get filtered images
-        pre_type_filters = copy(tmp_type_filters)
-        pre_type_filters.update(wftask.type_filters)
+        type_filters = copy(current_dataset_type_filters)
+        type_filters_patch = merge_type_filters(
+            task_input_types=task.input_types,
+            wftask_type_filters=wftask.type_filters,
+        )
+        type_filters.update(type_filters_patch)
         filtered_images = filter_image_list(
             images=tmp_images,
-            type_filters=pre_type_filters,
+            type_filters=type_filters,
             attribute_filters=job_attribute_filters,
         )
-        # Verify that filtered images comply with task input_types
-        for image in filtered_images:
-            if not match_filter(
-                image=image,
-                type_filters=task.input_types,
-                attribute_filters={},
-            ):
-                raise JobExecutionError(
-                    "Invalid filtered image list\n"
-                    f"Task input types: {task.input_types=}\n"
-                    f'Image zarr_url: {image["zarr_url"]}\n'
-                    f'Image types: {image["types"]}\n'
-                )
+
         # First, set status SUBMITTED in dataset.history for each wftask
         with next(get_sync_db()) as db:
             db_dataset = db.get(DatasetV2, dataset.id)
@@ -262,30 +254,9 @@ def execute_tasks_v2(
             else:
                 tmp_images.pop(img_search["index"])
 
-        # Update type_filters
-
-        # Assign the type filters based on different sources
-        # (task manifest and post-execution task output)
+        # Update type_filters based on task-manifest output_types
         type_filters_from_task_manifest = task.output_types
-        type_filters_from_task_output = current_task_output.type_filters
-
-        # Check that key sets are disjoint
-        keys_from_manifest = set(type_filters_from_task_manifest.keys())
-        keys_from_task_output = set(type_filters_from_task_output.keys())
-        if not keys_from_manifest.isdisjoint(keys_from_task_output):
-            overlap = keys_from_manifest.intersection(keys_from_task_output)
-            raise JobExecutionError(
-                "Some type filters are being set twice, "
-                f"for task '{task_name}'.\n"
-                f"Types from task output: {type_filters_from_task_output}\n"
-                "Types from task manifest: "
-                f"{type_filters_from_task_manifest}\n"
-                f"Overlapping keys: {overlap}"
-            )
-
-        # Update filters.types
-        tmp_type_filters.update(type_filters_from_task_manifest)
-        tmp_type_filters.update(type_filters_from_task_output)
+        current_dataset_type_filters.update(type_filters_from_task_manifest)
 
         # Write current dataset attributes (history, images, filters) into the
         # database. They can be used (1) to retrieve the latest state
@@ -294,7 +265,7 @@ def execute_tasks_v2(
         with next(get_sync_db()) as db:
             db_dataset = db.get(DatasetV2, dataset.id)
             db_dataset.history[-1]["status"] = WorkflowTaskStatusTypeV2.DONE
-            db_dataset.type_filters = tmp_type_filters
+            db_dataset.type_filters = current_dataset_type_filters
             db_dataset.images = tmp_images
             for attribute_name in [
                 "type_filters",

fractal_server/app/runner/v2/task_interface.py

@@ -1,31 +1,14 @@
 from typing import Any
-from typing import Optional
 
 from pydantic import BaseModel
 from pydantic import Extra
 from pydantic import Field
-from pydantic import root_validator
 from pydantic import validator
 
 from ....images import SingleImageTaskOutput
-from fractal_server.app.schemas._filter_validators import validate_type_filters
-from fractal_server.app.schemas._validators import root_validate_dict_keys
 from fractal_server.urls import normalize_url
 
 
-class LegacyFilters(BaseModel, extra=Extra.forbid):
-    """
-    For fractal-server<2.11, task output could include both
-    `filters["attributes"]` and `filters["types"]`. In the new version
-    there is a single field, named `type_filters`.
-    The current schema is only used to convert old type filters into the
-    new form, but it will reject any attribute filters.
-    """
-
-    types: dict[str, bool] = Field(default_factory=dict)
-    _types = validator("types", allow_reuse=True)(validate_type_filters)
-
-
 class TaskOutput(BaseModel, extra=Extra.forbid):
 
     image_list_updates: list[SingleImageTaskOutput] = Field(
@@ -33,16 +16,6 @@ class TaskOutput(BaseModel, extra=Extra.forbid):
     )
     image_list_removals: list[str] = Field(default_factory=list)
 
-    filters: Optional[LegacyFilters] = None
-    type_filters: dict[str, bool] = Field(default_factory=dict)
-
-    _dict_keys = root_validator(pre=True, allow_reuse=True)(
-        root_validate_dict_keys
-    )
-    _type_filters = validator("type_filters", allow_reuse=True)(
-        validate_type_filters
-    )
-
     def check_zarr_urls_are_unique(self) -> None:
         zarr_urls = [img.zarr_url for img in self.image_list_updates]
         zarr_urls.extend(self.image_list_removals)
@@ -62,20 +35,6 @@ class TaskOutput(BaseModel, extra=Extra.forbid):
                 msg = f"{msg}\n{duplicate}"
             raise ValueError(msg)
 
-    @root_validator()
-    def update_legacy_filters(cls, values):
-        if values["filters"] is not None:
-            if values["type_filters"] != {}:
-                raise ValueError(
-                    "Cannot set both (legacy) 'filters' and 'type_filters'."
-                )
-            else:
-                # Convert legacy filters.types into new type_filters
-                values["type_filters"] = values["filters"].types
-                values["filters"] = None
-
-        return values
-
     @validator("image_list_removals")
     def normalize_paths(cls, v: list[str]) -> list[str]:
         return [normalize_url(zarr_url) for zarr_url in v]

fractal_server/app/schemas/_filter_validators.py

@@ -34,7 +34,7 @@ def validate_attribute_filters(
             # values is a non-empty list, and its items must all be of the
             # same scalar non-None type
             _type = type(values[0])
-            if not all(isinstance(value, _type) for value in values):
+            if not all(type(value) is _type for value in values):
                 raise ValueError(
                     f"attribute_filters[{key}] has values with "
                     f"non-homogeneous types: {values}."

fractal_server/data_migrations/2_11_0.py

@@ -65,3 +65,5 @@ def fix_db():
 
         db.commit()
         logger.info("Changes committed.")
+
+    logger.info("END execution of fix_db function")

fractal_server/images/tools.py

@@ -98,3 +98,28 @@ def filter_image_list(
         )
     ]
     return filtered_images
+
+
+def merge_type_filters(
+    *,
+    task_input_types: dict[str, bool],
+    wftask_type_filters: dict[str, bool],
+) -> dict[str, bool]:
+    """
+    Merge two type-filters sets, if they are compatible.
+    """
+    all_keys = set(task_input_types.keys()) | set(wftask_type_filters.keys())
+    for key in all_keys:
+        if (
+            key in task_input_types.keys()
+            and key in wftask_type_filters.keys()
+            and task_input_types[key] != wftask_type_filters[key]
+        ):
+            raise ValueError(
+                "Cannot merge type filters "
+                f"`{task_input_types}` (from task) "
+                f"and `{wftask_type_filters}` (from workflowtask)."
+            )
+    merged_dict = task_input_types
+    merged_dict.update(wftask_type_filters)
+    return merged_dict

fractal_server/utils.py

@@ -107,18 +107,20 @@ def execute_command_sync(
     returncode = res.returncode
     stdout = res.stdout
     stderr = res.stderr
-    logger.debug(f"{returncode=}")
-    logger.debug("STDOUT:")
-    logger.debug(stdout)
-    logger.debug("STDERR:")
-    logger.debug(stderr)
     if res.returncode != 0:
         logger.debug(f"ERROR in subprocess call to '{command}'")
         raise RuntimeError(
             f"Command {command} failed.\n"
             f"returncode={res.returncode}\n"
-            f"{stdout=}\n"
-            f"{stderr=}\n"
+            "STDOUT:\n"
+            f"{stdout}\n"
+            "STDERR:\n"
+            f"{stderr}\n"
         )
+    logger.debug(f"{returncode=}")
+    logger.debug("STDOUT:")
+    logger.debug(stdout)
+    logger.debug("STDERR:")
+    logger.debug(stderr)
     logger.debug(f"END   subprocess call to '{command}'")
     return stdout

{fractal_server-2.11.0a3.dist-info → fractal_server-2.11.0a4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.11.0a3
+Version: 2.11.0a4
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.11.0a3.dist-info → fractal_server-2.11.0a4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=o_3QCs2FdlBclhmQ7GWAIcInUjrL307BrDoHMarxuSE,25
+fractal_server/__init__.py,sha256=DZWnRj7O86D5FPeOgamWSQFlyHVz7jh1SwTyyBfs6GQ,25
 fractal_server/__main__.py,sha256=D2YTmSowmXNyvqOjW_HeItCZT2UliWlySl_owicaZg0,8026
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -81,7 +81,7 @@ fractal_server/app/runner/exceptions.py,sha256=_qZ_t8O4umAdJ1ikockiF5rDJuxnEskrG
 fractal_server/app/runner/executors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/runner/executors/slurm/_batching.py,sha256=3mfeFuYm3UA4EXh4VWuqZTF-dcINECZgTHoPOaOszDo,8840
-fractal_server/app/runner/executors/slurm/_slurm_config.py,sha256=P0TDfIFf07h0hIhVNZUcY3t5vgdjptU-2T0uC_ZBEB4,15688
+fractal_server/app/runner/executors/slurm/_slurm_config.py,sha256=Qa5UgcMZfYEj95dMA5GF3WgNx8zQ_XH6ufNYYmhI4rs,15684
 fractal_server/app/runner/executors/slurm/remote.py,sha256=wLziIsGdSMiO-jIXM8x77JRK82g_2hx0iBKTiMghuIo,5852
 fractal_server/app/runner/executors/slurm/ssh/__init__.py,sha256=Cjn1rYvljddi96tAwS-qqGkNfOcfPzjChdaEZEObCcM,65
 fractal_server/app/runner/executors/slurm/ssh/_executor_wait_thread.py,sha256=bKo5Ja0IGxJWpPWyh9dN0AG-PwzTDZzD5LyaEHB3YU4,3742
@@ -127,14 +127,14 @@ fractal_server/app/runner/v2/_slurm_sudo/__init__.py,sha256=sGc1wGMPD6yQv6-MmTTG
 fractal_server/app/runner/v2/_slurm_sudo/_submit_setup.py,sha256=a5_FDPH_yxYmrjAjMRLgh_Y4DSG3mRslCLQodGM3-t4,2838
 fractal_server/app/runner/v2/deduplicate_list.py,sha256=-imwO7OB7ATADEnqVbTElUwoY0YIJCTf_SbWJNN9OZg,639
 fractal_server/app/runner/v2/handle_failed_job.py,sha256=-zFWw4d208bQEFUF_sAdH2LdHEARyg1FC8BENr1SjhU,2045
-fractal_server/app/runner/v2/merge_outputs.py,sha256=DW1d-fT0UcSnJUmz8xfU-AEeI7p_G0aQ_lNpDAe9C2o,1226
-fractal_server/app/runner/v2/runner.py,sha256=tdEjF2SWwDKXCHXz0wFRtKTRCG2I5BSjuqY08YSjMHs,12906
+fractal_server/app/runner/v2/merge_outputs.py,sha256=D1L4Taieq9i71SPQyNc1kMokgHh-sV_MqF3bv7QMDBc,907
+fractal_server/app/runner/v2/runner.py,sha256=ll0-nglUytoxDnMwMRgjY2901UnFb37bkSsxWK1KLA0,11595
 fractal_server/app/runner/v2/runner_functions.py,sha256=BLREIcQaE6FSc2AEJyZuiYk6rGazEz_9gprUqUZDljs,9488
 fractal_server/app/runner/v2/runner_functions_low_level.py,sha256=1fWvQ6YZUUnDhO_mipXC5hnaT-zK-GHxg8ayoxZX82k,3648
-fractal_server/app/runner/v2/task_interface.py,sha256=DU0fppTqvwhVkKwjPvfWPuZt09rV7AV2U5Vpu5CRqX8,3268
+fractal_server/app/runner/v2/task_interface.py,sha256=d6HPwPzrytUMVjExTU6fuCEwtvvWGRaSje0iXcRD45w,1728
 fractal_server/app/runner/versions.py,sha256=dSaPRWqmFPHjg20kTCHmi_dmGNcCETflDtDLronNanU,852
 fractal_server/app/schemas/__init__.py,sha256=stURAU_t3AOBaH0HSUbV-GKhlPKngnnIMoqWc3orFyI,135
-fractal_server/app/schemas/_filter_validators.py,sha256=9oGzf1yghFkIMbzluvxeRCjPei8V7dn7ot4ggQUmz8w,1731
+fractal_server/app/schemas/_filter_validators.py,sha256=WcfQ3ARc-2pj2oQFB4lWA0X5wtoOPGzpD4hJq4BblXs,1727
 fractal_server/app/schemas/_validators.py,sha256=3dotVxUHWKAmUO3aeoluYDLRKrw1OS-NxcZ4Fg_HOYk,3560
 fractal_server/app/schemas/user.py,sha256=icjox9gK_invW44Nh_L4CvqfRa92qghyQhmevyg09nQ,2243
 fractal_server/app/schemas/user_group.py,sha256=t30Kd07PY43G_AqFDb8vjdInTeLeU9WvFZDx8fVLPSI,1750
@@ -165,13 +165,13 @@ fractal_server/app/security/__init__.py,sha256=qn6idYgl-p5HWea0gTVnz4JnkoxGEkmQj
 fractal_server/app/security/signup_email.py,sha256=DrL51UdTSrgjleynMD5CRZwTSOpPrZ96fasRV0fvxDE,1165
 fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
 fractal_server/config.py,sha256=9rAzw7OO6ZeHEz-I8NJHuGoHf4xCHxfFLyRNZQD9ytY,27019
-fractal_server/data_migrations/2_11_0.py,sha256=PPFg1GxpfW5hElwzr_kx0_fYuEaNSyH23uCxztExO14,2411
+fractal_server/data_migrations/2_11_0.py,sha256=glS3BkhumrA6SpHiE_QFBgA7Bm2cbDCUlQyY3BjEub8,2464
 fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
 fractal_server/data_migrations/tools.py,sha256=LeMeASwYGtEqd-3wOLle6WARdTGAimoyMmRbbJl-hAM,572
 fractal_server/gunicorn_fractal.py,sha256=u6U01TLGlXgq1v8QmEpLih3QnsInZD7CqphgJ_GrGzc,1230
 fractal_server/images/__init__.py,sha256=-_wjoKtSX02P1KjDxDP_EXKvmbONTRmbf7iGVTsyBpM,154
 fractal_server/images/models.py,sha256=fAecChXhs4utRX4123Lgz5e_b_H0YtHrvNHCenR7tOs,3359
-fractal_server/images/tools.py,sha256=VFjm5pCOWjc2ms0t0s4nH6bK53ZU867JDL5oTEm4M1Q,2648
+fractal_server/images/tools.py,sha256=iqFx_pp46OoHsHjXxX6GrkXJPPfTo_c1WYvRur0olaE,3455
 fractal_server/logger.py,sha256=zwg_AjIHkNP0ruciXjm5lI5UFP3n6tMHullsM9lDjz4,5039
 fractal_server/main.py,sha256=gStLT9Du5QMpc9SyvRvtKU21EKwp-dG4HL3zGHzE06A,4908
 fractal_server/migrations/env.py,sha256=9t_OeKVlhM8WRcukmTrLbWNup-imiBGP_9xNgwCbtpI,2730
@@ -239,10 +239,10 @@ fractal_server/tasks/v2/utils_package_names.py,sha256=RDg__xrvQs4ieeVzmVdMcEh95v
 fractal_server/tasks/v2/utils_python_interpreter.py,sha256=5_wrlrTqXyo1YuLZvAW9hrSoh5MyLOzdPVUlUwM7uDQ,955
 fractal_server/tasks/v2/utils_templates.py,sha256=07TZpJ0Mh_A4lXVXrrH2o1VLFFGwxeRumA6DdgMgCWk,2947
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
-fractal_server/utils.py,sha256=utvmBx8K9I8hRWFquxna2pBaOqe0JifDL_NVPmihEJI,3525
+fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.11.0a3.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.11.0a3.dist-info/METADATA,sha256=PpEo4R-FpyRduTHTutRl1UdRHHwtb1dyFUKmLT2Zm1I,4564
-fractal_server-2.11.0a3.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fractal_server-2.11.0a3.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.11.0a3.dist-info/RECORD,,
+fractal_server-2.11.0a4.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.11.0a4.dist-info/METADATA,sha256=Y_VNqSmaDdR4YhuzzFtSRAbS4nPicovQyWytyGfKB18,4564
+fractal_server-2.11.0a4.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fractal_server-2.11.0a4.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.11.0a4.dist-info/RECORD,,