fractal-server 1.4.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

fractal_server/app/schemas/v2/workflowtask.py

@@ -0,0 +1,208 @@
+from enum import Enum
+from typing import Any
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import Field
+from pydantic import root_validator
+from pydantic import validator
+
+from .._validators import valdictkeys
+from .._validators import valint
+from ..v1.task import TaskExportV1
+from ..v1.task import TaskImportV1
+from .task import TaskExportV2
+from .task import TaskImportV2
+from .task import TaskLegacyReadV2
+from .task import TaskReadV2
+from fractal_server.images import Filters
+
+RESERVED_ARGUMENTS = {"zarr_dir", "zarr_url", "zarr_urls", "init_args"}
+
+
+class WorkflowTaskStatusTypeV2(str, Enum):
+    """
+    Define the available values for the status of a `WorkflowTask`.
+
+    This model is used within the `Dataset.history` attribute, which is
+    constructed in the runner and then used in the API (e.g. in the
+    `api/v2/project/{project_id}/dataset/{dataset_id}/status` endpoint).
+
+    Attributes:
+        SUBMITTED: The `WorkflowTask` is part of a running job.
+        DONE: The most-recent execution of this `WorkflowTask` was successful.
+        FAILED: The most-recent execution of this `WorkflowTask` failed.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
+
+    meta_non_parallel: Optional[dict[str, Any]]
+    meta_parallel: Optional[dict[str, Any]]
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+    order: Optional[int]
+    input_filters: Filters = Field(default_factory=Filters)
+
+    is_legacy_task: bool = False
+
+    # Validators
+    _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
+        valdictkeys("meta_non_parallel")
+    )
+    _meta_parallel = validator("meta_parallel", allow_reuse=True)(
+        valdictkeys("meta_parallel")
+    )
+    _order = validator("order", allow_reuse=True)(valint("order", min_val=0))
+
+    @validator("args_non_parallel")
+    def validate_args_non_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_non_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @validator("args_parallel")
+    def validate_args_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @root_validator
+    def validate_legacy_task(cls, values):
+        if values["is_legacy_task"] and (
+            values.get("meta_non_parallel") is not None
+            or values.get("args_non_parallel") is not None
+        ):
+            raise ValueError(
+                "If Task is legacy, 'args_non_parallel' and 'meta_non_parallel"
+                "must be None"
+            )
+        return values
+
+
+class WorkflowTaskReadV2(BaseModel):
+
+    id: int
+
+    workflow_id: int
+    order: Optional[int]
+    meta_non_parallel: Optional[dict[str, Any]]
+    meta_parallel: Optional[dict[str, Any]]
+
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+
+    input_filters: Filters
+
+    is_legacy_task: bool
+    task_type: str
+    task_id: Optional[int]
+    task: Optional[TaskReadV2]
+    task_legacy_id: Optional[int]
+    task_legacy: Optional[TaskLegacyReadV2]
+
+
+class WorkflowTaskUpdateV2(BaseModel):
+
+    meta_non_parallel: Optional[dict[str, Any]]
+    meta_parallel: Optional[dict[str, Any]]
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+    input_filters: Optional[Filters]
+
+    # Validators
+    _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
+        valdictkeys("meta_non_parallel")
+    )
+    _meta_parallel = validator("meta_parallel", allow_reuse=True)(
+        valdictkeys("meta_parallel")
+    )
+
+    @validator("args_non_parallel")
+    def validate_args_non_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_non_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @validator("args_parallel")
+    def validate_args_parallel(cls, value):
+        if value is None:
+            return
+        valdictkeys("args_parallel")(value)
+        args_keys = set(value.keys())
+        intersect_keys = RESERVED_ARGUMENTS.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+
+class WorkflowTaskImportV2(BaseModel):
+
+    meta_non_parallel: Optional[dict[str, Any]] = None
+    meta_parallel: Optional[dict[str, Any]] = None
+    args_non_parallel: Optional[dict[str, Any]] = None
+    args_parallel: Optional[dict[str, Any]] = None
+
+    input_filters: Optional[Filters] = None
+
+    is_legacy_task: bool = False
+    task: Optional[TaskImportV2] = None
+    task_legacy: Optional[TaskImportV1] = None
+
+    _meta_non_parallel = validator("meta_non_parallel", allow_reuse=True)(
+        valdictkeys("meta_non_parallel")
+    )
+    _meta_parallel = validator("meta_parallel", allow_reuse=True)(
+        valdictkeys("meta_parallel")
+    )
+    _args_non_parallel = validator("args_non_parallel", allow_reuse=True)(
+        valdictkeys("args_non_parallel")
+    )
+    _args_parallel = validator("args_parallel", allow_reuse=True)(
+        valdictkeys("args_parallel")
+    )
+
+
+class WorkflowTaskExportV2(BaseModel):
+
+    meta_non_parallel: Optional[dict[str, Any]] = None
+    meta_parallel: Optional[dict[str, Any]] = None
+    args_non_parallel: Optional[dict[str, Any]] = None
+    args_parallel: Optional[dict[str, Any]] = None
+    input_filters: Filters = Field(default_factory=Filters)
+
+    is_legacy_task: bool = False
+    task: Optional[TaskExportV2]
+    task_legacy: Optional[TaskExportV1]

fractal_server/config.py

@@ -347,6 +347,14 @@ class Settings(BaseSettings):
     with the same path and query parameters.
     """
 
+    FRACTAL_RUNNER_TASKS_INCLUDE_IMAGE: str = (
+        "Copy OME-Zarr structure;Convert Metadata Components from 2D to 3D"
+    )
+    """
+    `;`-separated list of names for task that require the `metadata["image"]`
+    attribute in their input-arguments JSON file.
+    """
+
     ###########################################################################
     # BUSINESS LOGIC
     ###########################################################################
@@ -382,12 +390,11 @@ class Settings(BaseSettings):
 
         info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
         if self.FRACTAL_RUNNER_BACKEND == "slurm":
-            try:
-                import cfut  # noqa: F401
-            except ModuleNotFoundError:
-                raise FractalConfigurationError(
-                    f"{info} but `clusterfutures` is not available"
-                )
+
+            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+                load_slurm_config_file,
+            )
+
             if not self.FRACTAL_SLURM_CONFIG_FILE:
                 raise FractalConfigurationError(
                     f"Must set FRACTAL_SLURM_CONFIG_FILE when {info}"
@@ -399,10 +406,6 @@ class Settings(BaseSettings):
                         f"{self.FRACTAL_SLURM_CONFIG_FILE} not found."
                     )
 
-                from fractal_server.app.runner._slurm._slurm_config import (
-                    load_slurm_config_file,
-                )
-
                 load_slurm_config_file(self.FRACTAL_SLURM_CONFIG_FILE)
                 if not shutil.which("sbatch"):
                     raise FractalConfigurationError(

fractal_server/images/__init__.py

@@ -0,0 +1,4 @@
+from .models import Filters  # noqa: F401
+from .models import SingleImage  # noqa: F401
+from .models import SingleImageTaskOutput  # noqa: F401
+from .models import SingleImageUpdate  # noqa: F401

fractal_server/images/models.py

@@ -0,0 +1,136 @@
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+from fractal_server.app.schemas._validators import valdictkeys
+from fractal_server.urls import normalize_url
+
+
+class SingleImageBase(BaseModel):
+    """
+    Base for SingleImage and SingleImageTaskOutput.
+
+    Attributes:
+        zarr_url:
+        origin:
+        attributes:
+        types:
+    """
+
+    zarr_url: str
+    origin: Optional[str] = None
+
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    # Validators
+    _attributes = validator("attributes", allow_reuse=True)(
+        valdictkeys("attributes")
+    )
+    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
+
+    @validator("zarr_url")
+    def normalize_zarr_url(cls, v: str) -> str:
+        return normalize_url(v)
+
+    @validator("origin")
+    def normalize_orig(cls, v: Optional[str]) -> Optional[str]:
+        if v is not None:
+            return normalize_url(v)
+
+
+class SingleImageTaskOutput(SingleImageBase):
+    """
+    `SingleImageBase`, with scalar `attributes` values (`None` included).
+    """
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool, None]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool, type(None))):
+                raise ValueError(
+                    f"SingleImageTaskOutput.attributes[{key}] must be a "
+                    "scalar (int, float, str or bool). "
+                    f"Given {value} ({type(value)})"
+                )
+        return v
+
+
+class SingleImage(SingleImageBase):
+    """
+    `SingleImageBase`, with scalar `attributes` values (`None` excluded).
+    """
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool)):
+                raise ValueError(
+                    f"SingleImage.attributes[{key}] must be a scalar "
+                    f"(int, float, str or bool). Given {value} ({type(value)})"
+                )
+        return v
+
+
+class SingleImageUpdate(BaseModel):
+    zarr_url: str
+    attributes: Optional[dict[str, Any]]
+    types: Optional[dict[str, bool]]
+
+    @validator("zarr_url")
+    def normalize_zarr_url(cls, v: str) -> str:
+        return normalize_url(v)
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool]]:
+        if v is not None:
+            # validate keys
+            valdictkeys("attributes")(v)
+            # validate values
+            for key, value in v.items():
+                if not isinstance(value, (int, float, str, bool)):
+                    raise ValueError(
+                        f"SingleImageUpdate.attributes[{key}] must be a scalar"
+                        " (int, float, str or bool). "
+                        f"Given {value} ({type(value)})"
+                    )
+        return v
+
+    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
+
+
+class Filters(BaseModel):
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    class Config:
+        extra = "forbid"
+
+    # Validators
+    _attributes = validator("attributes", allow_reuse=True)(
+        valdictkeys("attributes")
+    )
+    _types = validator("types", allow_reuse=True)(valdictkeys("types"))
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool, None]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool, type(None))):
+                raise ValueError(
+                    f"Filters.attributes[{key}] must be a scalar "
+                    "(int, float, str, bool, or None). "
+                    f"Given {value} ({type(value)})"
+                )
+        return v

fractal_server/images/tools.py

@@ -0,0 +1,84 @@
+from copy import copy
+from typing import Any
+from typing import Literal
+from typing import Optional
+from typing import Union
+
+from fractal_server.images import Filters
+
+
+ImageSearch = dict[Literal["image", "index"], Union[int, dict[str, Any]]]
+
+
+def find_image_by_zarr_url(
+    *,
+    images: list[dict[str, Any]],
+    zarr_url: str,
+) -> Optional[ImageSearch]:
+    """
+    Return a copy of the image with a given zarr_url, and its positional index.
+
+    Arguments:
+        images: List of images.
+        zarr_url: Path that the returned image must have.
+
+    Returns:
+        The first image from `images` which has zarr_url equal to `zarr_url`.
+    """
+    image_urls = [img["zarr_url"] for img in images]
+    try:
+        ind = image_urls.index(zarr_url)
+    except ValueError:
+        return None
+    return dict(image=copy(images[ind]), index=ind)
+
+
+def match_filter(image: dict[str, Any], filters: Filters) -> bool:
+    """
+    Find whether an image matches a filter set.
+
+    Arguments:
+        image: A single image.
+        filters: A set of filters.
+
+    Returns:
+        Whether the image matches the filter set.
+    """
+    # Verify match with types (using a False default)
+    for key, value in filters.types.items():
+        if image["types"].get(key, False) != value:
+            return False
+    # Verify match with attributes (only for non-None filters)
+    for key, value in filters.attributes.items():
+        if value is None:
+            continue
+        if image["attributes"].get(key) != value:
+            return False
+    return True
+
+
+def filter_image_list(
+    images: list[dict[str, Any]],
+    filters: Filters,
+) -> list[dict[str, Any]]:
+    """
+    Compute a sublist with images that match a filter set.
+
+    Arguments:
+        images: A list of images.
+        filters: A set of filters.
+
+    Returns:
+        List of the `images` elements which match the filter set.
+    """
+
+    # When no filter is provided, return all images
+    if filters.attributes == {} and filters.types == {}:
+        return images
+
+    filtered_images = [
+        copy(this_image)
+        for this_image in images
+        if match_filter(this_image, filters=filters)
+    ]
+    return filtered_images

fractal_server/main.py

@@ -32,13 +32,21 @@ def collect_routers(app: FastAPI) -> None:
     """
     from .app.routes.api import router_api
     from .app.routes.api.v1 import router_api_v1
-    from .app.routes.admin import router_admin
+    from .app.routes.api.v2 import router_api_v2
+    from .app.routes.admin.v1 import router_admin_v1
+    from .app.routes.admin.v2 import router_admin_v2
     from .app.routes.auth import router_auth
 
     app.include_router(router_api, prefix="/api")
     app.include_router(router_api_v1, prefix="/api/v1")
-    app.include_router(router_admin, prefix="/admin", tags=["Admin area"])
-    app.include_router(router_auth, prefix="/auth", tags=["auth"])
+    app.include_router(router_api_v2, prefix="/api/v2")
+    app.include_router(
+        router_admin_v1, prefix="/admin/v1", tags=["V1 Admin area"]
+    )
+    app.include_router(
+        router_admin_v2, prefix="/admin/v2", tags=["V2 Admin area"]
+    )
+    app.include_router(router_auth, prefix="/auth", tags=["Authentication"])
 
 
 def check_settings() -> None:

fractal_server/migrations/env.py

@@ -58,7 +58,6 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
-        render_as_batch=True,
     )
 
     with context.begin_transaction():
@@ -69,7 +68,6 @@ def do_run_migrations(connection: Connection) -> None:
     context.configure(
         connection=connection,
         target_metadata=target_metadata,
-        render_as_batch=True,
     )
 
     with context.begin_transaction():