fractal-server 2.11.0a9__py3-none-any.whl → 2.11.1__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "2.11.0a9"
+__VERSION__ = "2.11.1"

fractal_server/app/models/v2/dataset.py

@@ -1,6 +1,5 @@
 from datetime import datetime
 from typing import Any
-from typing import Literal
 from typing import Optional
 
 from sqlalchemy import Column
@@ -42,9 +41,6 @@ class DatasetV2(SQLModel, table=True):
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
-    filters: Optional[
-        dict[Literal["attributes", "types"], dict[str, Any]]
-    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
     type_filters: dict[str, bool] = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )

fractal_server/app/models/v2/workflowtask.py

@@ -1,5 +1,4 @@
 from typing import Any
-from typing import Literal
 from typing import Optional
 
 from sqlalchemy import Column
@@ -25,9 +24,6 @@ class WorkflowTaskV2(SQLModel, table=True):
     args_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
     args_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
 
-    input_filters: Optional[
-        dict[Literal["attributes", "types"], dict[str, Any]]
-    ] = Field(sa_column=Column(JSON, nullable=True, server_default="null"))
     type_filters: dict[str, bool] = Field(
         sa_column=Column(JSON, nullable=False, server_default="{}")
     )

fractal_server/app/routes/api/v2/job.py

@@ -1,4 +1,6 @@
+import asyncio
 from pathlib import Path
+from typing import Iterator
 from typing import Optional
 
 from fastapi import APIRouter
@@ -24,6 +26,14 @@ from ._aux_functions import _get_workflow_check_owner
 from fractal_server.app.models import UserOAuth
 from fractal_server.app.routes.auth import current_active_user
 
+
+# https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread
+# This moves the function execution to a separate thread,
+# preventing it from blocking the event loop.
+async def zip_folder_threaded(folder: str) -> Iterator[bytes]:
+    return await asyncio.to_thread(_zip_folder_to_byte_stream_iterator, folder)
+
+
 router = APIRouter()
 
 
@@ -128,8 +138,11 @@ async def download_job_logs(
     )
     job = output["job"]
     zip_name = f"{Path(job.working_dir).name}_archive.zip"
+
+    zip_bytes_iterator = await zip_folder_threaded(job.working_dir)
+
     return StreamingResponse(
-        _zip_folder_to_byte_stream_iterator(folder=job.working_dir),
+        zip_bytes_iterator,
         media_type="application/x-zip-compressed",
         headers={"Content-Disposition": f"attachment;filename={zip_name}"},
     )

fractal_server/app/schemas/v2/workflowtask.py

@@ -180,7 +180,7 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
     args_non_parallel: Optional[dict[str, Any]] = None
     args_parallel: Optional[dict[str, Any]] = None
     type_filters: Optional[dict[str, bool]] = None
-    filters: Optional[dict[str, Any]] = None
+    input_filters: Optional[dict[str, Any]] = None
 
     task: Union[TaskImportV2, TaskImportV2Legacy]
 
@@ -191,7 +191,7 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
         Transform legacy filters (created with fractal-server<2.11.0)
         into type filters
         """
-        if values.get("filters") is not None:
+        if values.get("input_filters") is not None:
             if "type_filters" in values.keys():
                 raise ValueError(
                     "Cannot set filters both through the legacy field "
@@ -201,13 +201,15 @@ class WorkflowTaskImportV2(BaseModel, extra=Extra.forbid):
             else:
                 # As of 2.11.0, WorkflowTask do not have attribute filters
                 # any more.
-                if values["filters"]["attributes"] != {}:
+                if values["input_filters"]["attributes"] != {}:
                     raise ValueError(
                         "Cannot set attribute filters for WorkflowTasks."
                     )
                 # Convert legacy filters.types into new type_filters
-                values["type_filters"] = values["filters"].get("types", {})
-                values["filters"] = None
+                values["type_filters"] = values["input_filters"].get(
+                    "types", {}
+                )
+                values["input_filters"] = None
 
         return values
 

fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py

@@ -0,0 +1,54 @@
+"""drop old filter columns
+
+Revision ID: af8673379a5c
+Revises: db09233ad13a
+Create Date: 2025-01-30 14:44:04.302795
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "af8673379a5c"
+down_revision = "db09233ad13a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.drop_column("filters")
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_column("input_filters")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "input_filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'null'::json"),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+
+    with op.batch_alter_table("datasetv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "filters",
+                postgresql.JSON(astext_type=sa.Text()),
+                server_default=sa.text("'null'::json"),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+
+    # ### end Alembic commands ###

{fractal_server-2.11.0a9.dist-info → fractal_server-2.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: fractal-server
-Version: 2.11.0a9
+Version: 2.11.1
 Summary: Backend component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause

{fractal_server-2.11.0a9.dist-info → fractal_server-2.11.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fractal_server/__init__.py,sha256=Cq36Bza-Whq5ZsTq5ttVGI_RHoX8KyV7NfthnFizHn8,25
+fractal_server/__init__.py,sha256=IDMsn_8tp36mX2SfY8u--GT2UfFEEyrK2bOGt3Pg-e0,23
 fractal_server/__main__.py,sha256=D2YTmSowmXNyvqOjW_HeItCZT2UliWlySl_owicaZg0,8026
 fractal_server/alembic.ini,sha256=MWwi7GzjzawI9cCAK1LW7NxIBQDUqD12-ptJoq5JpP0,3153
 fractal_server/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -16,13 +16,13 @@ fractal_server/app/models/v1/state.py,sha256=m9gMZqqnm3oDpJNJp-Lht4kM7oO7pcEI7sL
 fractal_server/app/models/v1/task.py,sha256=uFXam7eu3Ye1Yt7_g7llCzY8BetmDRilsq5hR2C1Zbg,2640
 fractal_server/app/models/v1/workflow.py,sha256=dnY5eMaOe3oZv8arn00RNX9qVkBtTLG-vYdWXcQuyo4,3950
 fractal_server/app/models/v2/__init__.py,sha256=63THGEZQlxWcosGCI74SEvJU7wOoOn1j1byTjf4NFOI,526
-fractal_server/app/models/v2/dataset.py,sha256=gO_V_Fsmw44uB1Nc6MLBVOGxQeutxpDi8nvhhwkggbE,1696
+fractal_server/app/models/v2/dataset.py,sha256=RuqTHXWEgs4A3OSk8Pq9DTq9Xr7w1IJNnyXhOzrDiR0,1509
 fractal_server/app/models/v2/job.py,sha256=BMmu5oXdZvN7jEIAMZvQMB3PQBcCYzxn6Qm6HdRWre4,1725
 fractal_server/app/models/v2/project.py,sha256=rAHoh5KfYwIaW7rTX0_O0jvWmxEvfo1BafvmcXuSSRk,786
 fractal_server/app/models/v2/task.py,sha256=jebD28Pz8tGcsWCItxj6uKjcD8BMMnnU8dqYhvhEB6c,1520
 fractal_server/app/models/v2/task_group.py,sha256=Sd-fb7EN18eOxrS-RT4ekczLWp-tQcbX5C4LrcmjoIM,3443
 fractal_server/app/models/v2/workflow.py,sha256=YBgFGCziUgU0aJ5EM3Svu9W2c46AewZO9VBlFCHiSps,1069
-fractal_server/app/models/v2/workflowtask.py,sha256=nidHo87GNgMVvL2WDm0HOfGLwGY3m65GTDAInwbqk1Q,1312
+fractal_server/app/models/v2/workflowtask.py,sha256=sBnKiEx9KqhprUaZVY6R8lyfCdwRaX8obYm6bXdn6_E,1119
 fractal_server/app/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/routes/admin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fractal_server/app/routes/admin/v1.py,sha256=ggJZMeKhRijfVe2h2VzfIcpR15FqkKImANhkTXl0mSk,12908
@@ -48,7 +48,7 @@ fractal_server/app/routes/api/v2/_aux_functions_task_lifecycle.py,sha256=c8eqPXd
 fractal_server/app/routes/api/v2/_aux_functions_tasks.py,sha256=uhNSs-jcS7ndIUFKiOC1yrDiViw3uvKEXi9UL04BMks,11642
 fractal_server/app/routes/api/v2/dataset.py,sha256=Y6uZz--YSEGgnPYu05rZ9sr1Ug08bNl2v1h3VeApBe8,9441
 fractal_server/app/routes/api/v2/images.py,sha256=0qkItqPrAvWEaK3YHUmCCrKrO_tQuzAPf4Te0q8mON8,8832
-fractal_server/app/routes/api/v2/job.py,sha256=Bga2Kz1OjvDIdxZObWaaXVhNIhC_5JKhKRjEH2_ayEE,5157
+fractal_server/app/routes/api/v2/job.py,sha256=m89FTh9Px25oXCeWj2k2NdGWQaO2oxMh-6lZppcsJOY,5551
 fractal_server/app/routes/api/v2/project.py,sha256=eWYFJ7F2ZYQcpi-_n-rhPF-Q4gJhzYBsVGYFhHZZXAE,6653
 fractal_server/app/routes/api/v2/status.py,sha256=_cDZW-ESYw6zpf-lLFFqko5bLpKhqKrCM6yv1OfqxN4,6300
 fractal_server/app/routes/api/v2/submit.py,sha256=UMPhWwk4FqZmYtVEu6WLPkSr6a2R4wwgPPeVrx0zRME,8622
@@ -160,12 +160,11 @@ fractal_server/app/schemas/v2/task.py,sha256=9W4xe-p19pbLpjsNJrYVIMpkrlwuCxGAJNO
 fractal_server/app/schemas/v2/task_collection.py,sha256=9c_yyFcVBXdAZpQQniy1bROhYnQT7G1BflOpMY1joPE,6250
 fractal_server/app/schemas/v2/task_group.py,sha256=zZfvAH7c3MZC4_An09TMuOkNE_e1Z9XsYEnmN-axHdU,3217
 fractal_server/app/schemas/v2/workflow.py,sha256=-KWvXnbHBFA3pj5n7mfSyLKJQSqkJmoziIEe7mpLl3M,1875
-fractal_server/app/schemas/v2/workflowtask.py,sha256=RLAoKIpc7WFYp20N3Pr6lUu6BkOHpPf4Yvs77FH_Uzg,7658
+fractal_server/app/schemas/v2/workflowtask.py,sha256=xjFTmnKuHSetJvN-9k_GTMbPKwPg-J00zUkc_96QO7E,7726
 fractal_server/app/security/__init__.py,sha256=qn6idYgl-p5HWea0gTVnz4JnkoxGEkmQjPzvKpDWT0I,14035
 fractal_server/app/security/signup_email.py,sha256=DrL51UdTSrgjleynMD5CRZwTSOpPrZ96fasRV0fvxDE,1165
 fractal_server/app/user_settings.py,sha256=OP1yiYKtPadxwM51_Q0hdPk3z90TCN4z1BLpQsXyWiU,1316
 fractal_server/config.py,sha256=9rAzw7OO6ZeHEz-I8NJHuGoHf4xCHxfFLyRNZQD9ytY,27019
-fractal_server/data_migrations/2_11_0.py,sha256=Cc-P_eXKnlQxc2AnRzV0K4B4DDeEHEFm1WltFjtRMnA,6400
 fractal_server/data_migrations/README.md,sha256=_3AEFvDg9YkybDqCLlFPdDmGJvr6Tw7HRI14aZ3LOIw,398
 fractal_server/data_migrations/tools.py,sha256=LeMeASwYGtEqd-3wOLle6WARdTGAimoyMmRbbJl-hAM,572
 fractal_server/gunicorn_fractal.py,sha256=u6U01TLGlXgq1v8QmEpLih3QnsInZD7CqphgJ_GrGzc,1230
@@ -196,6 +195,7 @@ fractal_server/migrations/versions/99ea79d9e5d2_add_dataset_history.py,sha256=0i
 fractal_server/migrations/versions/9c5ae74c9b98_add_user_settings_table.py,sha256=syONdZNf4-OnAcWIsbzXpYwpXPsXZ4SsmjwVvmVG0PU,2256
 fractal_server/migrations/versions/9fd26a2b0de4_add_workflow_timestamp_created.py,sha256=4l1AHGUsa0ONoJVZlr3fTXw_xbbQ8O7wlD92Az2aRfM,1849
 fractal_server/migrations/versions/a7f4d6137b53_add_workflow_dump_to_applyworkflow.py,sha256=ekDUML7ILpmdoqEclKbEUdyLi4uw9HSG_sTjG2hp_JE,867
+fractal_server/migrations/versions/af8673379a5c_drop_old_filter_columns.py,sha256=9sLd0F7nO5chHHm7RZ4wBA-9bvWomS-av_odKwODADM,1551
 fractal_server/migrations/versions/d256a7379ab8_taskgroup_activity_and_venv_info_to_.py,sha256=HN3_Pk8G81SzdYjg4K1RZAyjKSlsZGvcYE2nWOUbwxQ,3861
 fractal_server/migrations/versions/d4fe3708d309_make_applyworkflow_workflow_dump_non_.py,sha256=6cHEZFuTXiQg9yu32Y3RH1XAl71av141WQ6UMbiITIg,949
 fractal_server/migrations/versions/da2cb2ac4255_user_group_viewer_paths.py,sha256=yGWSA2HIHUybcVy66xBITk08opV2DFYSCIIrulaUZhI,901
@@ -241,8 +241,8 @@ fractal_server/tasks/v2/utils_templates.py,sha256=07TZpJ0Mh_A4lXVXrrH2o1VLFFGwxe
 fractal_server/urls.py,sha256=QjIKAC1a46bCdiPMu3AlpgFbcv6a4l3ABcd5xz190Og,471
 fractal_server/utils.py,sha256=PMwrxWFxRTQRl1b9h-NRIbFGPKqpH_hXnkAT3NfZdpY,3571
 fractal_server/zip_tools.py,sha256=GjDgo_sf6V_DDg6wWeBlZu5zypIxycn_l257p_YVKGc,4876
-fractal_server-2.11.0a9.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
-fractal_server-2.11.0a9.dist-info/METADATA,sha256=xTEE_9-GBVNq4xonqeNLqXtQ62YJvN9IR8ExYa_L9bc,4564
-fractal_server-2.11.0a9.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-fractal_server-2.11.0a9.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
-fractal_server-2.11.0a9.dist-info/RECORD,,
+fractal_server-2.11.1.dist-info/LICENSE,sha256=QKAharUuhxL58kSoLizKJeZE3mTCBnX6ucmz8W0lxlk,1576
+fractal_server-2.11.1.dist-info/METADATA,sha256=2-8AQ21JsDrgiHyn28rTFVP8-viNbAJu9rJxBC5z2nA,4562
+fractal_server-2.11.1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+fractal_server-2.11.1.dist-info/entry_points.txt,sha256=8tV2kynvFkjnhbtDnxAqImL6HMVKsopgGfew0DOp5UY,58
+fractal_server-2.11.1.dist-info/RECORD,,

fractal_server/data_migrations/2_11_0.py

@@ -1,168 +0,0 @@
-import logging
-from typing import Union
-
-from sqlalchemy.orm.attributes import flag_modified
-from sqlmodel import select
-
-from fractal_server.app.db import get_sync_db
-from fractal_server.app.models import DatasetV2
-from fractal_server.app.models import JobV2
-from fractal_server.app.models import ProjectV2
-from fractal_server.app.models import WorkflowTaskV2
-from fractal_server.app.models import WorkflowV2
-from fractal_server.app.schemas.v2 import DatasetReadV2
-from fractal_server.app.schemas.v2 import JobReadV2
-from fractal_server.app.schemas.v2 import ProjectReadV2
-from fractal_server.app.schemas.v2 import TaskReadV2
-from fractal_server.app.schemas.v2 import WorkflowTaskReadV2
-from fractal_server.images.models import AttributeFiltersType
-
-logger = logging.getLogger("fix_db")
-logger.setLevel(logging.INFO)
-
-
-def dict_values_to_list(
-    input_dict: dict[str, Union[int, float, bool, str, None]],
-    identifier: str,
-) -> tuple[AttributeFiltersType, bool]:
-    was_there_a_warning = False
-    for k, v in input_dict.items():
-        if not isinstance(v, (int, float, bool, str, type(None))):
-            error_msg = (
-                f"Attribute '{k}' from '{identifier}' "
-                "has invalid type '{type(v)}'."
-            )
-            logger.error(error_msg)
-            raise RuntimeError(error_msg)
-        elif v is None:
-            logger.warning(
-                f"Attribute '{k}' from '{identifier}' is "
-                "None and it will be removed."
-            )
-            was_there_a_warning = True
-        else:
-            input_dict[k] = [v]
-    return input_dict, was_there_a_warning
-
-
-def fix_db():
-    logger.info("START execution of fix_db function")
-
-    with next(get_sync_db()) as db:
-        # DatasetV2.filters
-        stm = select(DatasetV2).order_by(DatasetV2.id)
-        datasets = db.execute(stm).scalars().all()
-        for ds in datasets:
-            logger.info(f"DatasetV2[{ds.id}] START")
-            if ds.filters is None:
-                logger.info(f"DatasetV2[{ds.id}] SKIP")
-                continue
-
-            ds.attribute_filters, warning = dict_values_to_list(
-                ds.filters["attributes"],
-                f"Dataset[{ds.id}].filters.attributes",
-            )
-            if warning:
-                proj = db.get(ProjectV2, ds.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{ds.id=}, "
-                    f"{ds.name=}"
-                )
-            ds.type_filters = ds.filters["types"]
-            ds.filters = None
-            for i, h in enumerate(ds.history):
-                ds.history[i]["workflowtask"]["type_filters"] = h[
-                    "workflowtask"
-                ]["input_filters"]["types"]
-                ds.history[i]["workflowtask"].pop("input_filters")
-            flag_modified(ds, "history")
-            DatasetReadV2(
-                **ds.model_dump(),
-                project=ProjectReadV2(**ds.project.model_dump()),
-            )
-            db.add(ds)
-            logger.info(f"DatasetV2[{ds.id}] END - fixed filters")
-
-        logger.info("------ switch from dataset to workflowtasks ------")
-
-        # WorkflowTaskV2.input_filters
-        stm = select(WorkflowTaskV2).order_by(WorkflowTaskV2.id)
-        wftasks = db.execute(stm).scalars().all()
-        for wft in wftasks:
-            logger.info(f"WorkflowTaskV2[{wft.id}] START")
-            if wft.input_filters is None:
-                logger.info(f"WorkflowTaskV2[{wft.id}] SKIP")
-                continue
-            wft.type_filters = wft.input_filters["types"]
-            if wft.input_filters["attributes"]:
-                logger.warning(
-                    "Removing input_filters['attributes']. "
-                    f"(previous value: {wft.input_filters['attributes']})"
-                )
-                wf = db.get(WorkflowV2, wft.workflow_id)
-                proj = db.get(ProjectV2, wf.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{wf.id=}, "
-                    f"{wf.name=}, "
-                    f"{wft.task.name=}"
-                )
-            wft.input_filters = None
-            flag_modified(wft, "input_filters")
-            WorkflowTaskReadV2(
-                **wft.model_dump(),
-                task=TaskReadV2(**wft.task.model_dump()),
-            )
-            db.add(wft)
-            logger.info(f"WorkflowTaskV2[{wft.id}] END - fixed filters")
-
-        logger.info("------ switch from workflowtasks to jobs ------")
-
-        # JOBS V2
-        stm = select(JobV2).order_by(JobV2.id)
-        jobs = db.execute(stm).scalars().all()
-        for job in jobs:
-            logger.info(f"JobV2[{job.id}] START")
-            if "filters" not in job.dataset_dump.keys():
-                logger.info(f"JobV2[{job.id}] SKIP")
-                continue
-            job.dataset_dump["type_filters"] = job.dataset_dump["filters"][
-                "types"
-            ]
-            (
-                job.dataset_dump["attribute_filters"],
-                warning,
-            ) = dict_values_to_list(
-                job.dataset_dump["filters"]["attributes"],
-                f"JobV2[{job.id}].dataset_dump.filters.attributes",
-            )
-            if warning and job.project_id is not None:
-                proj = db.get(ProjectV2, job.project_id)
-                logger.warning(
-                    "Additional information: "
-                    f"{proj.id=}, "
-                    f"{proj.name=}, "
-                    f"{proj.user_list[0].email=}, "
-                    f"{job.id=}, "
-                    f"{job.start_timestamp=}, "
-                    f"{job.end_timestamp=}, "
-                    f"{job.dataset_id=}, "
-                    f"{job.workflow_id=}."
-                )
-            job.dataset_dump.pop("filters")
-            flag_modified(job, "dataset_dump")
-            JobReadV2(**job.model_dump())
-            db.add(job)
-            logger.info(f"JobV2[{job.id}] END - fixed filters")
-
-        db.commit()
-        logger.info("Changes committed.")
-
-    logger.info("END execution of fix_db function")