fractal-server 2.0.0a0__py3-none-any.whl → 2.0.0a2__py3-none-any.whl

fractal_server/app/runner/v2/runner.py

@@ -8,7 +8,7 @@ from typing import Optional
 
 from ....images import Filters
 from ....images import SingleImage
-from ....images.tools import _filter_image_list
+from ....images.tools import filter_image_list
 from ....images.tools import find_image_by_path
 from ....images.tools import match_filter
 from ..filenames import FILTERS_FILENAME
@@ -52,16 +52,15 @@ def execute_tasks_v2(
         # PRE TASK EXECUTION
 
         # Get filtered images
-        pre_type_filters = copy(tmp_filters["types"])
-        pre_type_filters.update(wftask.input_filters["types"])
-        pre_attribute_filters = copy(tmp_filters["attributes"])
-        pre_attribute_filters.update(wftask.input_filters["attributes"])
-        filtered_images = _filter_image_list(
+        pre_filters = dict(
+            types=copy(tmp_filters["types"]),
+            attributes=copy(tmp_filters["attributes"]),
+        )
+        pre_filters["types"].update(wftask.input_filters["types"])
+        pre_filters["attributes"].update(wftask.input_filters["attributes"])
+        filtered_images = filter_image_list(
             images=tmp_images,
-            filters=Filters(
-                types=pre_type_filters,
-                attributes=pre_attribute_filters,
-            ),
+            filters=Filters(**pre_filters),
         )
         # Verify that filtered images comply with task input_types
         for image in filtered_images:
@@ -183,22 +182,26 @@ def execute_tasks_v2(
                 updated_attributes.update(image["attributes"])
                 updated_types.update(image["types"])
                 updated_types.update(task.output_types)
-                new_image = SingleImage(
+                new_image = dict(
                     path=image["path"],
                     origin=image["origin"],
                     attributes=updated_attributes,
                     types=updated_types,
                 )
+                # Validate new image
+                SingleImage(**new_image)
                 # Add image into the dataset image list
-                tmp_images.append(new_image.dict())
+                tmp_images.append(new_image)
 
         # Remove images from tmp_images
-        for image in current_task_output.image_list_removals:
+        for image_path in current_task_output.image_list_removals:
             image_search = find_image_by_path(
-                images=tmp_images, path=image["path"]
+                images=tmp_images, path=image_path
             )
-            if image_search["index"] is None:
-                raise
+            if image_search is None:
+                raise ValueError(
+                    f"Cannot remove missing image with path {image_path=}"
+                )
             else:
                 tmp_images.pop(image_search["index"])
 

fractal_server/app/runner/v2/runner_functions.py

@@ -1,24 +1,24 @@
 import functools
+import logging
 import traceback
 from concurrent.futures import Executor
 from pathlib import Path
+from typing import Any
 from typing import Callable
 from typing import Literal
 from typing import Optional
 
-from ....images import SingleImage
 from .deduplicate_list import deduplicate_list
 from .merge_outputs import merge_outputs
 from .runner_functions_low_level import run_single_task
-from .task_interface import InitArgsModel
 from .task_interface import InitTaskOutput
 from .task_interface import TaskOutput
 from .v1_compat import convert_v2_args_into_v1
 from fractal_server.app.models.v1 import Task as TaskV1
 from fractal_server.app.models.v2 import TaskV2
 from fractal_server.app.models.v2 import WorkflowTaskV2
-from fractal_server.app.runner.v2.components import _COMPONENT_KEY_
-from fractal_server.app.runner.v2.components import _index_to_component
+from fractal_server.app.runner.components import _COMPONENT_KEY_
+from fractal_server.app.runner.components import _index_to_component
 
 
 __all__ = [
@@ -80,7 +80,7 @@ def _check_parallelization_list_size(my_list):
 
 def run_v2_task_non_parallel(
     *,
-    images: list[SingleImage],
+    images: list[dict[str, Any]],
     zarr_dir: str,
     task: TaskV2,
     wftask: WorkflowTaskV2,
@@ -94,7 +94,11 @@ def run_v2_task_non_parallel(
     This runs server-side (see `executor` argument)
     """
 
-    if not workflow_dir_user:
+    if workflow_dir_user is None:
+        workflow_dir_user = workflow_dir
+        logging.warning(
+            "In `run_single_task`, workflow_dir_user=None. Is this right?"
+        )
         workflow_dir_user = workflow_dir
 
     executor_options = _get_executor_options(
@@ -132,7 +136,7 @@ def run_v2_task_non_parallel(
 
 def run_v2_task_parallel(
     *,
-    images: list[SingleImage],
+    images: list[dict[str, Any]],
     task: TaskV2,
     wftask: WorkflowTaskV2,
     executor: Executor,
@@ -191,7 +195,7 @@ def run_v2_task_parallel(
 
 def run_v2_task_compound(
     *,
-    images: list[SingleImage],
+    images: list[dict[str, Any]],
     zarr_dir: str,
     task: TaskV2,
     wftask: WorkflowTaskV2,
@@ -240,9 +244,7 @@ def run_v2_task_compound(
     else:
         init_task_output = InitTaskOutput(**output)
     parallelization_list = init_task_output.parallelization_list
-    parallelization_list = deduplicate_list(
-        parallelization_list, PydanticModel=InitArgsModel
-    )
+    parallelization_list = deduplicate_list(parallelization_list)
 
     # 3/B: parallel part of a compound task
     _check_parallelization_list_size(parallelization_list)
@@ -287,7 +289,7 @@ def run_v2_task_compound(
 
 def run_v1_task_parallel(
     *,
-    images: list[SingleImage],
+    images: list[dict[str, Any]],
     task_legacy: TaskV1,
     wftask: WorkflowTaskV2,
     executor: Executor,

fractal_server/app/runner/v2/runner_functions_low_level.py

@@ -7,9 +7,9 @@ from shlex import split as shlex_split
 from typing import Any
 from typing import Optional
 
+from ..components import _COMPONENT_KEY_
 from ..exceptions import JobExecutionError
 from ..exceptions import TaskExecutionError
-from .components import _COMPONENT_KEY_
 from fractal_server.app.models.v2 import WorkflowTaskV2
 from fractal_server.app.runner.task_files import get_task_file_paths
 

fractal_server/app/schemas/v2/dataset.py

@@ -2,6 +2,7 @@ from datetime import datetime
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
@@ -40,7 +41,7 @@ class DatasetStatusReadV2(BaseModel):
 # CRUD
 
 
-class DatasetCreateV2(BaseModel):
+class DatasetCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
 

fractal_server/app/schemas/v2/job.py

@@ -3,6 +3,7 @@ from enum import Enum
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import validator
 from pydantic.types import StrictStr
 
@@ -34,7 +35,7 @@ class JobStatusTypeV2(str, Enum):
     FAILED = "failed"
 
 
-class JobCreateV2(BaseModel):
+class JobCreateV2(BaseModel, extra=Extra.forbid):
 
     first_task_index: Optional[int] = None
     last_task_index: Optional[int] = None

fractal_server/app/schemas/v2/manifest.py

@@ -43,13 +43,63 @@ class TaskManifestV2(BaseModel):
     executable_parallel: Optional[str] = None
     input_types: dict[str, bool] = Field(default_factory=dict)
     output_types: dict[str, bool] = Field(default_factory=dict)
-    meta_parallel: dict[str, Any] = Field(default_factory=dict)
     meta_non_parallel: dict[str, Any] = Field(default_factory=dict)
+    meta_parallel: dict[str, Any] = Field(default_factory=dict)
     args_schema_non_parallel: Optional[dict[str, Any]] = None
     args_schema_parallel: Optional[dict[str, Any]] = None
     docs_info: Optional[str] = None
     docs_link: Optional[HttpUrl] = None
 
+    @root_validator
+    def validate_executable_args_meta(cls, values):
+
+        executable_non_parallel = values.get("executable_non_parallel")
+        executable_parallel = values.get("executable_parallel")
+        if (executable_non_parallel is None) and (executable_parallel is None):
+
+            raise ValueError(
+                "`TaskManifestV2.executable_non_parallel` and "
+                "`TaskManifestV2.executable_parallel` cannot be both None."
+            )
+
+        elif executable_non_parallel is None:
+
+            meta_non_parallel = values.get("meta_non_parallel")
+            if meta_non_parallel != {}:
+                raise ValueError(
+                    "`TaskManifestV2.meta_non_parallel` must be an empty dict "
+                    "if `TaskManifestV2.executable_non_parallel` is None. "
+                    f"Given: {meta_non_parallel}."
+                )
+
+            args_schema_non_parallel = values.get("args_schema_non_parallel")
+            if args_schema_non_parallel is not None:
+                raise ValueError(
+                    "`TaskManifestV2.args_schema_non_parallel` must be None "
+                    "if `TaskManifestV2.executable_non_parallel` is None. "
+                    f"Given: {args_schema_non_parallel}."
+                )
+
+        elif executable_parallel is None:
+
+            meta_parallel = values.get("meta_parallel")
+            if meta_parallel != {}:
+                raise ValueError(
+                    "`TaskManifestV2.meta_parallel` must be an empty dict if "
+                    "`TaskManifestV2.executable_parallel` is None. "
+                    f"Given: {meta_parallel}."
+                )
+
+            args_schema_parallel = values.get("args_schema_parallel")
+            if args_schema_parallel is not None:
+                raise ValueError(
+                    "`TaskManifestV2.args_schema_parallel` must be None if "
+                    "`TaskManifestV2.executable_parallel` is None. "
+                    f"Given: {args_schema_parallel}."
+                )
+
+        return values
+
 
 class ManifestV2(BaseModel):
     """

fractal_server/app/schemas/v2/project.py

@@ -2,13 +2,14 @@ from datetime import datetime
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
 from .._validators import valutc
 
 
-class ProjectCreateV2(BaseModel):
+class ProjectCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
     read_only: bool = False

fractal_server/app/schemas/v2/task.py

@@ -3,6 +3,7 @@ from typing import Literal
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import Field
 from pydantic import HttpUrl
 from pydantic import root_validator
@@ -11,9 +12,7 @@ from pydantic import validator
 from .._validators import valstr
 
 
-class TaskCreateV2(BaseModel):
-    class Config:
-        extra = "forbid"
+class TaskCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
 

fractal_server/app/schemas/v2/workflow.py

@@ -2,6 +2,7 @@ from datetime import datetime
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import validator
 
 from .._validators import valstr
@@ -12,7 +13,7 @@ from .workflowtask import WorkflowTaskImportV2
 from .workflowtask import WorkflowTaskReadV2
 
 
-class WorkflowCreateV2(BaseModel):
+class WorkflowCreateV2(BaseModel, extra=Extra.forbid):
 
     name: str
 

fractal_server/app/schemas/v2/workflowtask.py

@@ -3,6 +3,7 @@ from typing import Any
 from typing import Optional
 
 from pydantic import BaseModel
+from pydantic import Extra
 from pydantic import Field
 from pydantic import validator
 
@@ -35,7 +36,7 @@ class WorkflowTaskStatusTypeV2(str, Enum):
     FAILED = "failed"
 
 
-class WorkflowTaskCreateV2(BaseModel):
+class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
 
     meta_parallel: Optional[dict[str, Any]]
     meta_non_parallel: Optional[dict[str, Any]]

fractal_server/images/__init__.py

@@ -1,50 +1,2 @@
-from typing import Any
-from typing import Optional
-from typing import Union
-
-from pydantic import BaseModel
-from pydantic import Field
-from pydantic import validator
-
-
-class SingleImage(BaseModel):
-
-    path: str
-    origin: Optional[str] = None
-
-    attributes: dict[str, Any] = Field(default_factory=dict)
-    types: dict[str, bool] = Field(default_factory=dict)
-
-    @validator("attributes")
-    def validate_attributes(
-        cls, v: dict[str, Any]
-    ) -> dict[str, Union[int, float, str, bool]]:
-        for key, value in v.items():
-            if not isinstance(value, (int, float, str, bool)):
-                raise ValueError(
-                    f"SingleImage.attributes[{key}] must be a scalar "
-                    f"(int, float, str or bool). Given {value} ({type(value)})"
-                )
-        return v
-
-
-class Filters(BaseModel):
-    attributes: dict[str, Any] = Field(default_factory=dict)
-    types: dict[str, bool] = Field(default_factory=dict)
-
-    class Config:
-        extra = "forbid"
-
-    # Validators
-    @validator("attributes")
-    def validate_attributes(
-        cls, v: dict[str, Any]
-    ) -> dict[str, Union[int, float, str, bool, None]]:
-        for key, value in v.items():
-            if not isinstance(value, (int, float, str, bool, type(None))):
-                raise ValueError(
-                    f"Filters.attributes[{key}] must be a scalar "
-                    "(int, float, str, bool, or None). "
-                    f"Given {value} ({type(value)})"
-                )
-        return v
+from .models import Filters  # noqa: F401
+from .models import SingleImage  # noqa: F401

fractal_server/images/models.py

@@ -0,0 +1,50 @@
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+
+class SingleImage(BaseModel):
+
+    path: str
+    origin: Optional[str] = None
+
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool)):
+                raise ValueError(
+                    f"SingleImage.attributes[{key}] must be a scalar "
+                    f"(int, float, str or bool). Given {value} ({type(value)})"
+                )
+        return v
+
+
+class Filters(BaseModel):
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    class Config:
+        extra = "forbid"
+
+    # Validators
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool, None]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool, type(None))):
+                raise ValueError(
+                    f"Filters.attributes[{key}] must be a scalar "
+                    "(int, float, str, bool, or None). "
+                    f"Given {value} ({type(value)})"
+                )
+        return v

fractal_server/images/tools.py

@@ -1,21 +1,24 @@
 from copy import copy
 from typing import Any
+from typing import Literal
 from typing import Optional
 from typing import Union
 
 from fractal_server.images import Filters
-from fractal_server.images import SingleImage
+
+
+ImageSearch = dict[Literal["image", "index"], Union[int, dict[str, Any]]]
 
 
 def find_image_by_path(
     *,
     images: list[dict[str, Any]],
     path: str,
-) -> Optional[dict[str, Union[int, dict[str, Any]]]]:
+) -> Optional[ImageSearch]:
     """
-    Return a copy of the image with a given path and its positional index.
+    Return a copy of the image with a given path, and its positional index.
 
-    Args:
+    Arguments:
         images: List of images.
         path: Path that the returned image must have.
 
@@ -30,10 +33,23 @@ def find_image_by_path(
     return dict(image=copy(images[ind]), index=ind)
 
 
-def match_filter(image: dict[str, Any], filters: Filters):
+# FIXME: what is filters
+def match_filter(image: dict[str, Any], filters: Filters) -> bool:
+    """
+    Find whether an image matches a filter set.
+
+    Arguments:
+        image: A single image.
+        filters: A set of filters.
+
+    Returns:
+        Whether the image matches the filter set.
+    """
+    # Verify match with types (using a False default)
     for key, value in filters.types.items():
         if image["types"].get(key, False) != value:
             return False
+    # Verify match with attributes (only for non-None filters)
     for key, value in filters.attributes.items():
         if value is None:
             continue
@@ -42,45 +58,28 @@ def match_filter(image: dict[str, Any], filters: Filters):
     return True
 
 
-def _filter_image_list(
+def filter_image_list(
     images: list[dict[str, Any]],
     filters: Filters,
 ) -> list[dict[str, Any]]:
+    """
+    Compute a sublist with images that match a filter set.
 
-    # When no filter is provided, return all images
-    if filters.attributes == {} and filters.types == {}:
-        return images
-
-    filtered_images = []
-    for this_image in images:
-        if match_filter(this_image, filters=filters):
-            filtered_images.append(copy(this_image))
-    return filtered_images
-
-
-def match_filter_SingleImage(image: SingleImage, filters: Filters):
-    for key, value in filters.types.items():
-        if image.types.get(key, False) != value:
-            return False
-    for key, value in filters.attributes.items():
-        if value is None:
-            continue
-        if image.attributes.get(key) != value:
-            return False
-    return True
-
+    Arguments:
+        images: A list of images.
+        filters: A set of filters.
 
-def _filter_image_list_SingleImage(
-    images: list[SingleImage],
-    filters: Filters,
-) -> list[SingleImage]:
+    Returns:
+        List of the `images` elements which match the filter set.
+    """
 
     # When no filter is provided, return all images
     if filters.attributes == {} and filters.types == {}:
         return images
 
-    filtered_images = []
-    for this_image in images:
-        if match_filter_SingleImage(this_image, filters=filters):
-            filtered_images.append(copy(this_image))
+    filtered_images = [
+        copy(this_image)
+        for this_image in images
+        if match_filter(this_image, filters=filters)
+    ]
     return filtered_images

fractal_server/migrations/env.py

@@ -58,7 +58,6 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
-        render_as_batch=True,
     )
 
     with context.begin_transaction():
@@ -69,7 +68,6 @@ def do_run_migrations(connection: Connection) -> None:
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
-        render_as_batch=True,
     )
 
     with context.begin_transaction():

fractal_server/migrations/versions/{56af171b0159_v2.py → d71e732236cd_v2.py}

@@ -1,8 +1,8 @@
 """v2
 
-Revision ID: 56af171b0159
+Revision ID: d71e732236cd
 Revises: 9fd26a2b0de4
-Create Date: 2024-03-22 11:09:02.458011
+Create Date: 2024-04-05 11:09:17.639183
 
 """
 import sqlalchemy as sa
@@ -11,7 +11,7 @@ from alembic import op
 
 
 # revision identifiers, used by Alembic.
-revision = "56af171b0159"
+revision = "d71e732236cd"
 down_revision = "9fd26a2b0de4"
 branch_labels = None
 depends_on = None
@@ -33,6 +33,7 @@ def upgrade() -> None:
         "taskv2",
         sa.Column("id", sa.Integer(), nullable=False),
         sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
         sa.Column(
             "command_non_parallel",
             sqlmodel.sql.sqltypes.AutoString(),
@@ -46,12 +47,18 @@ def upgrade() -> None:
         sa.Column(
             "source", sqlmodel.sql.sqltypes.AutoString(), nullable=False
         ),
-        sa.Column("meta", sa.JSON(), nullable=True),
+        sa.Column(
+            "meta_non_parallel", sa.JSON(), server_default="{}", nullable=False
+        ),
+        sa.Column(
+            "meta_parallel", sa.JSON(), server_default="{}", nullable=False
+        ),
         sa.Column("owner", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
         sa.Column(
             "version", sqlmodel.sql.sqltypes.AutoString(), nullable=True
         ),
-        sa.Column("args_schema", sa.JSON(), nullable=True),
+        sa.Column("args_schema_non_parallel", sa.JSON(), nullable=True),
+        sa.Column("args_schema_parallel", sa.JSON(), nullable=True),
         sa.Column(
             "args_schema_version",
             sqlmodel.sql.sqltypes.AutoString(),
@@ -177,8 +184,10 @@ def upgrade() -> None:
         sa.Column("id", sa.Integer(), nullable=False),
         sa.Column("workflow_id", sa.Integer(), nullable=False),
         sa.Column("order", sa.Integer(), nullable=True),
-        sa.Column("meta", sa.JSON(), nullable=True),
-        sa.Column("args", sa.JSON(), nullable=True),
+        sa.Column("meta_parallel", sa.JSON(), nullable=True),
+        sa.Column("meta_non_parallel", sa.JSON(), nullable=True),
+        sa.Column("args_parallel", sa.JSON(), nullable=True),
+        sa.Column("args_non_parallel", sa.JSON(), nullable=True),
         sa.Column(
             "input_filters",
             sa.JSON(),
@@ -186,6 +195,9 @@ def upgrade() -> None:
             nullable=False,
         ),
         sa.Column("is_legacy_task", sa.Boolean(), nullable=False),
+        sa.Column(
+            "task_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False
+        ),
         sa.Column("task_id", sa.Integer(), nullable=True),
         sa.Column("task_legacy_id", sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(
@@ -202,11 +214,21 @@ def upgrade() -> None:
         ),
         sa.PrimaryKeyConstraint("id"),
     )
+    op.add_column(
+        "task",
+        sa.Column(
+            "is_v2_compatible",
+            sa.Boolean(),
+            server_default=sa.text("false"),
+            nullable=False,
+        ),
+    )
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("task", "is_v2_compatible")
     op.drop_table("workflowtaskv2")
     op.drop_table("jobv2")
     op.drop_table("workflowv2")

fractal_server/tasks/v2/background_operations.py

@@ -352,7 +352,6 @@ async def background_collect_pip(
             data.status = "OK"
             data.log = get_collection_log(venv_path)
             state.data = data.sanitised_dict()
-            db.add(state)
             db.merge(state)
             db.commit()
 

{fractal_server-2.0.0a0.dist-info → fractal_server-2.0.0a2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fractal-server
-Version: 2.0.0a0
+Version: 2.0.0a2
 Summary: Server component of the Fractal analytics platform
 Home-page: https://github.com/fractal-analytics-platform/fractal-server
 License: BSD-3-Clause