fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/images/tools.py

@@ -0,0 +1,86 @@
+from copy import copy
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from fractal_server.images import Filters
+from fractal_server.images import SingleImage
+
+
+def find_image_by_path(
+    *,
+    images: list[dict[str, Any]],
+    path: str,
+) -> Optional[dict[str, Union[int, dict[str, Any]]]]:
+    """
+    Return a copy of the image with a given path and its positional index.
+
+    Args:
+        images: List of images.
+        path: Path that the returned image must have.
+
+    Returns:
+        The first image from `images` which has path equal to `path`.
+    """
+    image_paths = [img["path"] for img in images]
+    try:
+        ind = image_paths.index(path)
+    except ValueError:
+        return None
+    return dict(image=copy(images[ind]), index=ind)
+
+
+def match_filter(image: dict[str, Any], filters: Filters):
+    for key, value in filters.types.items():
+        if image["types"].get(key, False) != value:
+            return False
+    for key, value in filters.attributes.items():
+        if value is None:
+            continue
+        if image["attributes"].get(key) != value:
+            return False
+    return True
+
+
+def _filter_image_list(
+    images: list[dict[str, Any]],
+    filters: Filters,
+) -> list[dict[str, Any]]:
+
+    # When no filter is provided, return all images
+    if filters.attributes == {} and filters.types == {}:
+        return images
+
+    filtered_images = []
+    for this_image in images:
+        if match_filter(this_image, filters=filters):
+            filtered_images.append(copy(this_image))
+    return filtered_images
+
+
+def match_filter_SingleImage(image: SingleImage, filters: Filters):
+    for key, value in filters.types.items():
+        if image.types.get(key, False) != value:
+            return False
+    for key, value in filters.attributes.items():
+        if value is None:
+            continue
+        if image.attributes.get(key) != value:
+            return False
+    return True
+
+
+def _filter_image_list_SingleImage(
+    images: list[SingleImage],
+    filters: Filters,
+) -> list[SingleImage]:
+
+    # When no filter is provided, return all images
+    if filters.attributes == {} and filters.types == {}:
+        return images
+
+    filtered_images = []
+    for this_image in images:
+        if match_filter_SingleImage(this_image, filters=filters):
+            filtered_images.append(copy(this_image))
+    return filtered_images

fractal_server/main.py

@@ -32,13 +32,21 @@ def collect_routers(app: FastAPI) -> None:
     """
     from .app.routes.api import router_api
     from .app.routes.api.v1 import router_api_v1
-    from .app.routes.admin import router_admin
+    from .app.routes.api.v2 import router_api_v2
+    from .app.routes.admin.v1 import router_admin_v1
+    from .app.routes.admin.v2 import router_admin_v2
     from .app.routes.auth import router_auth
 
     app.include_router(router_api, prefix="/api")
     app.include_router(router_api_v1, prefix="/api/v1")
-    app.include_router(router_admin, prefix="/admin", tags=["Admin area"])
-    app.include_router(router_auth, prefix="/auth", tags=["auth"])
+    app.include_router(router_api_v2, prefix="/api/v2")
+    app.include_router(
+        router_admin_v1, prefix="/admin/v1", tags=["V1 Admin area"]
+    )
+    app.include_router(
+        router_admin_v2, prefix="/admin/v2", tags=["V2 Admin area"]
+    )
+    app.include_router(router_auth, prefix="/auth", tags=["Authentication"])
 
 
 def check_settings() -> None:

fractal_server/migrations/versions/4b35c5cefbe3_tmp_is_v2_compatible.py

@@ -0,0 +1,39 @@
+"""TMP - is_v2_compatible
+
+Revision ID: 4b35c5cefbe3
+Revises: 876f28db9d4e
+Create Date: 2024-03-28 15:26:33.436724
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "4b35c5cefbe3"
+down_revision = "876f28db9d4e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("task", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "is_v2_compatible",
+                sa.Boolean(),
+                server_default=sa.text("false"),
+                nullable=False,
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("task", schema=None) as batch_op:
+        batch_op.drop_column("is_v2_compatible")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/56af171b0159_v2.py

@@ -0,0 +1,217 @@
+"""v2
+
+Revision ID: 56af171b0159
+Revises: 9fd26a2b0de4
+Create Date: 2024-03-22 11:09:02.458011
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "56af171b0159"
+down_revision = "9fd26a2b0de4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "projectv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("read_only", sa.Boolean(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "taskv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column(
+            "command_non_parallel",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column(
+            "command_parallel",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column(
+            "source", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column("owner", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.Column(
+            "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column("args_schema", sa.JSON(), nullable=True),
+        sa.Column(
+            "args_schema_version",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column(
+            "docs_info", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "docs_link", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column("input_types", sa.JSON(), nullable=True),
+        sa.Column("output_types", sa.JSON(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("source"),
+    )
+    op.create_table(
+        "datasetv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("history", sa.JSON(), server_default="[]", nullable=False),
+        sa.Column("read_only", sa.Boolean(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column(
+            "zarr_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("images", sa.JSON(), server_default="[]", nullable=False),
+        sa.Column(
+            "filters",
+            sa.JSON(),
+            server_default='{"attributes": {}, "types": {}}',
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projectv2.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "linkuserprojectv2",
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projectv2.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["user_id"],
+            ["user_oauth.id"],
+        ),
+        sa.PrimaryKeyConstraint("project_id", "user_id"),
+    )
+    op.create_table(
+        "workflowv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.Column(
+            "timestamp_created", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projectv2.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "jobv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("project_id", sa.Integer(), nullable=True),
+        sa.Column("workflow_id", sa.Integer(), nullable=True),
+        sa.Column("dataset_id", sa.Integer(), nullable=True),
+        sa.Column(
+            "user_email", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column(
+            "slurm_account", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column("dataset_dump", sa.JSON(), nullable=False),
+        sa.Column("workflow_dump", sa.JSON(), nullable=False),
+        sa.Column("project_dump", sa.JSON(), nullable=False),
+        sa.Column(
+            "worker_init", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "working_dir", sqlmodel.sql.sqltypes.AutoString(), nullable=True
+        ),
+        sa.Column(
+            "working_dir_user",
+            sqlmodel.sql.sqltypes.AutoString(),
+            nullable=True,
+        ),
+        sa.Column("first_task_index", sa.Integer(), nullable=False),
+        sa.Column("last_task_index", sa.Integer(), nullable=False),
+        sa.Column(
+            "start_timestamp", sa.DateTime(timezone=True), nullable=False
+        ),
+        sa.Column("end_timestamp", sa.DateTime(timezone=True), nullable=True),
+        sa.Column(
+            "status", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
+        sa.Column("log", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["dataset_id"],
+            ["datasetv2.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["project_id"],
+            ["projectv2.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflow_id"],
+            ["workflowv2.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_table(
+        "workflowtaskv2",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("workflow_id", sa.Integer(), nullable=False),
+        sa.Column("order", sa.Integer(), nullable=True),
+        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column("args", sa.JSON(), nullable=True),
+        sa.Column(
+            "input_filters",
+            sa.JSON(),
+            server_default='{"attributes": {}, "types": {}}',
+            nullable=False,
+        ),
+        sa.Column("is_legacy_task", sa.Boolean(), nullable=False),
+        sa.Column("task_id", sa.Integer(), nullable=True),
+        sa.Column("task_legacy_id", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["task_id"],
+            ["taskv2.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["task_legacy_id"],
+            ["task.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflow_id"],
+            ["workflowv2.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("workflowtaskv2")
+    op.drop_table("jobv2")
+    op.drop_table("workflowv2")
+    op.drop_table("linkuserprojectv2")
+    op.drop_table("datasetv2")
+    op.drop_table("taskv2")
+    op.drop_table("projectv2")
+    # ### end Alembic commands ###

fractal_server/migrations/versions/876f28db9d4e_tmp_split_task_and_wftask_meta.py

@@ -0,0 +1,68 @@
+"""TMP - split Task and WFTask meta
+
+Revision ID: 876f28db9d4e
+Revises: b9e9eed9d442
+Create Date: 2024-03-27 14:35:50.076725
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "876f28db9d4e"
+down_revision = "b9e9eed9d442"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("meta_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("meta_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("meta")
+
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("meta_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("meta_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("meta")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "meta",
+                postgresql.JSON(astext_type=sa.Text()),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+        batch_op.drop_column("meta_non_parallel")
+        batch_op.drop_column("meta_parallel")
+
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "meta",
+                postgresql.JSON(astext_type=sa.Text()),
+                autoincrement=False,
+                nullable=True,
+            )
+        )
+        batch_op.drop_column("meta_parallel")
+        batch_op.drop_column("meta_non_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/974c802f0dd0_tmp_workflowtaskv2_type_in_db.py

@@ -0,0 +1,37 @@
+"""TMP - WorkflowTaskV2.type in DB
+
+Revision ID: 974c802f0dd0
+Revises: e3e639454d4b
+Create Date: 2024-04-02 11:21:20.212530
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "974c802f0dd0"
+down_revision = "e3e639454d4b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "task_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.drop_column("task_type")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/9cd305cd6023_tmp_workflowtaskv2.py

@@ -0,0 +1,40 @@
+"""TMP - WorkflowTaskV2
+
+Revision ID: 9cd305cd6023
+Revises: a6231ed6273c
+Create Date: 2024-03-26 09:15:00.188036
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision = "9cd305cd6023"
+down_revision = "a6231ed6273c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("args_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("workflowtaskv2", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("args", sqlite.JSON(), nullable=True))
+        batch_op.drop_column("args_non_parallel")
+        batch_op.drop_column("args_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/a6231ed6273c_tmp_args_schemas_in_taskv2.py

@@ -0,0 +1,42 @@
+"""TMP - Args Schemas in TaskV2
+
+Revision ID: a6231ed6273c
+Revises: 56af171b0159
+Create Date: 2024-03-26 08:49:51.870087
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import sqlite
+
+# revision identifiers, used by Alembic.
+revision = "a6231ed6273c"
+down_revision = "56af171b0159"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_schema_non_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.add_column(
+            sa.Column("args_schema_parallel", sa.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args_schema")
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column("args_schema", sqlite.JSON(), nullable=True)
+        )
+        batch_op.drop_column("args_schema_parallel")
+        batch_op.drop_column("args_schema_non_parallel")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/b9e9eed9d442_tmp_taskv2_type.py

@@ -0,0 +1,37 @@
+"""TMP - TaskV2.type
+
+Revision ID: b9e9eed9d442
+Revises: 9cd305cd6023
+Create Date: 2024-03-27 13:10:34.125503
+
+"""
+import sqlalchemy as sa
+import sqlmodel
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "b9e9eed9d442"
+down_revision = "9cd305cd6023"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+            )
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.drop_column("type")
+
+    # ### end Alembic commands ###

fractal_server/migrations/versions/e3e639454d4b_tmp_make_task_meta_non_optional.py

@@ -0,0 +1,50 @@
+"""TMP - make task.meta non optional
+
+Revision ID: e3e639454d4b
+Revises: 4b35c5cefbe3
+Create Date: 2024-03-29 17:10:57.643561
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e3e639454d4b"
+down_revision = "4b35c5cefbe3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+        )
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=False,
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("taskv2", schema=None) as batch_op:
+        batch_op.alter_column(
+            "meta_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+        )
+        batch_op.alter_column(
+            "meta_non_parallel",
+            existing_type=postgresql.JSON(astext_type=sa.Text()),
+            nullable=True,
+        )
+
+    # ### end Alembic commands ###

fractal_server/tasks/__init__.py

@@ -1,8 +1,3 @@
 """
 `tasks` module
-
-Submodules:
-    * [Endpoint functions](background_operations)
-    * [Background-task functions](endpoint_operations)
-    * [Auxiliary functions](utils)
 """